Merge branch 'develop' into hotfix/taos-tools

* develop: (66 commits)
  Delete client.log0.0
  not start mqtt module by default, fot it not finished yet
  add one line back for double insurance
  [TD-364] broken link is not handled in right way
  Add mqtt plugin to subscribe to telemetry data
  remove extra case list print out.
  scripts
  remove some log
  add randome insert many records with NULL then commit.
  [TD-388] drop dnode in community version
  add precalculation interface
  [td-225] update the test script
  [td-225] fix bugs.
  fix general/insert/insert_drop.sim error
  add error log when have to call pthread_cancel
  change forcestop() to stop() to emulate ctrl-c
  fix TD-394
  add test case to test insert then import scenario.
  jenkins
  fix duplicate table bug
  ...

.gitignore

@@ -41,3 +41,27 @@ pysim/
 
 # Doxygen Generated files
 html/
+/.vs
+/CMakeFiles/3.10.2
+/CMakeCache.txt
+/Makefile
+/*.cmake
+/deps
+/src/cq/test/CMakeFiles/cqtest.dir/*.cmake
+*.cmake
+/src/cq/test/CMakeFiles/cqtest.dir/*.make
+*.make
+link.txt
+*.internal
+*.includecache
+*.marks
+Makefile
+CMakeError.log
+*.log
+/CMakeFiles/CMakeRuleHashes.txt
+/CMakeFiles/Makefile2
+/CMakeFiles/TargetDirectories.txt
+/CMakeFiles/cmake.check_cache
+/out/isenseconfig/WSL-Clang-Debug
+/out/isenseconfig/WSL-GCC-Debug
+/test/cfg

.travis.yml

@@ -90,6 +90,7 @@ matrix:
         esac
 
   - os: linux
+    dist: bionic
     language: c
     compiler: gcc
     env: COVERITY_SCAN=true
@@ -125,6 +126,7 @@ matrix:
         branch_pattern: coverity_scan
 
   - os: linux
+    dist: bionic
     language: c
     compiler: gcc
     env: ENV_COVER=true
@@ -230,6 +232,7 @@ matrix:
       - make > /dev/null
 
   - os: linux
+    dist: bionic
     language: c
     compiler: clang
     env: DESC="linux/clang build"

CMakeSettings.json

+{
+  "configurations": [
+    {
+      "name": "WSL-GCC-Debug",
+      "generator": "Unix Makefiles",
+      "configurationType": "Debug",
+      "buildRoot": "${projectDir}\\build\\",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeExecutable": "/usr/bin/cmake",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "inheritEnvironments": [ "linux_x64" ],
+      "wslPath": "${defaultWSLPath}",
+      "addressSanitizerRuntimeFlags": "detect_leaks=0",
+      "variables": [
+        {
+          "name": "CMAKE_INSTALL_PREFIX",
+          "value": "/mnt/d/TDengine/TDengine/build",
+          "type": "PATH"
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file

src/client/inc/tsclient.h

@@ -226,11 +226,8 @@ typedef struct {
   int     command;
   uint8_t msgType;
 
-  union {
-    bool   existsCheck;     // check if the table exists or not
-    bool   autoCreated;     // if the table is missing, on-the-fly create it. during getmeterMeta
-    int8_t dataSourceType;  // load data from file or not
-  };
+  bool   autoCreated;        // if the table is missing, on-the-fly create it. during getmeterMeta
+  int8_t dataSourceType;     // load data from file or not
 
   union {
     int32_t count;

src/client/src/tscLocal.c

@@ -56,7 +56,7 @@ static int32_t getToStringLength(const char *pData, int32_t length, int32_t type
     } break;
     case TSDB_DATA_TYPE_TIMESTAMP:
     case TSDB_DATA_TYPE_BIGINT:
-      len = sprintf(buf, "%" PRId64 "", *(int64_t *)pData);
+      len = sprintf(buf, "%" PRId64, *(int64_t *)pData);
       break;
     case TSDB_DATA_TYPE_BOOL:
       len = MAX_BOOL_TYPE_LENGTH;

src/client/src/tscParseInsert.c

@@ -42,35 +42,35 @@ enum {
 static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows);
 
 static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) {
-  int32_t numType = isValidNumber(pToken);
-  if (TK_ILLEGAL == numType) {
-    return numType;
-  }
+//  int32_t numType = isValidNumber(pToken);
+//  if (TK_ILLEGAL == numType) {
+//    return numType;
+//  }
 
   int32_t radix = 10;
-  if (numType == TK_HEX) {
+  if (pToken->type == TK_HEX) {
     radix = 16;
-  } else if (numType == TK_OCT) {
+  } else if (pToken->type == TK_OCT) {
     radix = 8;
-  } else if (numType == TK_BIN) {
+  } else if (pToken->type == TK_BIN) {
     radix = 2;
   }
 
   errno = 0;
   *value = strtoll(pToken->z, endPtr, radix);
 
-  return numType;
+  return pToken->type;
 }
 
 static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) {
-  int32_t numType = isValidNumber(pToken);
-  if (TK_ILLEGAL == numType) {
-    return numType;
-  }
+//  int32_t numType = isValidNumber(pToken);
+//  if (TK_ILLEGAL == numType) {
+//    return numType;
+//  }
 
   errno = 0;
   *value = strtod(pToken->z, endPtr);
-  return numType;
+  return pToken->type;
 }
 
 int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) {
@@ -779,7 +779,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
 
     STagData *pTag = (STagData *)pCmd->payload;
     memset(pTag, 0, sizeof(STagData));
-    pCmd->payloadLen = sizeof(STagData);
     
     /*
      * the source super table is moved to the secondary position of the pTableMetaInfo list
@@ -928,6 +927,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       }
     }
 
+    // 3. calculate the actual data size of STagData
+    pCmd->payloadLen = sizeof(pTag->name) + sizeof(pTag->dataLen);
+    for (int32_t t = 0; t < numOfTags; ++t) {
+      pTag->dataLen += pTagSchema[t].bytes;
+      pCmd->payloadLen += pTagSchema[t].bytes;
+    }
+    pTag->dataLen = htonl(pTag->dataLen);
+
     if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) {
       return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr);
     }
@@ -949,7 +956,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
     } else {
       sql = sToken.z;
     }
-    code = tscGetTableMeta(pSql, pTableMetaInfo);
+    code = tscGetMeterMetaEx(pSql, pTableMetaInfo, false);
     
     if (pCmd->curSql == NULL) {
       assert(code == TSDB_CODE_ACTION_IN_PROGRESS);

src/client/src/tscProfile.c

@@ -23,8 +23,6 @@
 void  tscSaveSlowQueryFp(void *handle, void *tmrId);
 void *tscSlowQueryConn = NULL;
 bool  tscSlowQueryConnInitialized = false;
-TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
-                     void *param, void **taos);
 
 void tscInitConnCb(void *param, TAOS_RES *result, int code) {
   char *sql = param;

src/client/src/tscSQLParser.c

@@ -4416,6 +4416,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         TSDB_CODE_SUCCESS) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg13);
     }
+    pAlterSQL->tagData.dataLen = pTagsSchema->bytes;
 
     // validate the length of binary
     if ((pTagsSchema->type == TSDB_DATA_TYPE_BINARY || pTagsSchema->type == TSDB_DATA_TYPE_NCHAR) &&
@@ -4539,11 +4540,13 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
     return TSDB_CODE_INVALID_SQL;
   }
 
-  const SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[14] = {
-      {"resetLog", 8},      {"resetQueryCache", 15}, {"dDebugFlag", 10},       {"rpcDebugFlag", 12},
-      {"tmrDebugFlag", 12}, {"cDebugFlag", 10},      {"uDebugFlag", 10},       {"mDebugFlag", 10},
-      {"sdbDebugFlag", 12}, {"httpDebugFlag", 13},   {"monitorDebugFlag", 16}, {"qDebugflag", 10},
-      {"debugFlag", 9},     {"monitor", 7}};
+  const int DNODE_DYNAMIC_CFG_OPTIONS_SIZE = 17;
+  const SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[] = {
+      {"resetLog", 8},       {"resetQueryCache", 15},  {"debugFlag", 9},     {"mDebugFlag", 10},
+      {"dDebugFlag", 10},    {"sdbDebugFlag", 12},     {"vDebugFlag", 10},   {"cDebugFlag", 10},
+      {"httpDebugFlag", 13}, {"monitorDebugFlag", 16}, {"rpcDebugFlag", 12}, {"uDebugFlag", 10},
+      {"tmrDebugFlag", 12},  {"qDebugflag", 10},       {"sDebugflag", 10},   {"tsdbDebugFlag", 13},
+      {"monitor", 7}};
 
   SSQLToken* pOptionToken = &pOptions->a[1];
 
@@ -4555,8 +4558,8 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
         return TSDB_CODE_SUCCESS;
       }
     }
-  } else if ((strncasecmp(DNODE_DYNAMIC_CFG_OPTIONS[13].name, pOptionToken->z, pOptionToken->n) == 0) &&
-             (DNODE_DYNAMIC_CFG_OPTIONS[13].len == pOptionToken->n)) {
+  } else if ((strncasecmp(DNODE_DYNAMIC_CFG_OPTIONS[DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1].name, pOptionToken->z, pOptionToken->n) == 0) &&
+             (DNODE_DYNAMIC_CFG_OPTIONS[DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1].len == pOptionToken->n)) {
     SSQLToken* pValToken = &pOptions->a[2];
     int32_t    val = strtol(pValToken->z, NULL, 10);
     if (val != 0 && val != 1) {
@@ -4572,7 +4575,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
       return TSDB_CODE_INVALID_SQL;
     }
 
-    for (int32_t i = 2; i < tListLen(DNODE_DYNAMIC_CFG_OPTIONS) - 1; ++i) {
+    for (int32_t i = 2; i < DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1; ++i) {
       const SDNodeDynConfOption* pOption = &DNODE_DYNAMIC_CFG_OPTIONS[i];
 
       if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) {
@@ -5550,11 +5553,11 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
   // too long tag values will return invalid sql, not be truncated automatically
   SSchema* pTagSchema = tscGetTableTagSchema(pStableMeterMetaInfo->pTableMeta);
 
-  char* tagVal = pCreateTable->usingInfo.tagdata.data;
+  STagData* pTag = &pCreateTable->usingInfo.tagdata;
+  char* tagVal = pTag->data;
   int32_t ret = TSDB_CODE_SUCCESS;
   
   for (int32_t i = 0; i < pList->nExpr; ++i) {
-    
     if (pTagSchema[i].type == TSDB_DATA_TYPE_BINARY || pTagSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
       // validate the length of binary
       if (pList->a[i].pVar.nLen + VARSTR_HEADER_SIZE > pTagSchema[i].bytes) {
@@ -5593,6 +5596,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
     return ret;
   }
 
+  pTag->dataLen = tagVal - pTag->data;
   return TSDB_CODE_SUCCESS;
 }
 

src/client/src/tscSecondaryMerge.c

@@ -638,7 +638,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
   SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-  (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * 1);
+  (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pSql->numOfSubs);
   if (*pMemBuffer == NULL) {
     tscError("%p failed to allocate memory", pSql);
     pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
@@ -808,18 +808,19 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource *
   }
 }
 
-void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQueryInfo,
-                                       SFillInfo *pFillInfo) {
+void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQueryInfo, SFillInfo *pFillInfo) {
   // discard following dataset in the same group and reset the interpolation information
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
   STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
 
-  int16_t prec = tinfo.precision;
-  int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey;
-  int64_t revisedSTime =
-      taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec);
-  taosResetFillInfo(pFillInfo, revisedSTime);
+  if (pFillInfo != NULL) {
+    int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey;
+    int64_t revisedSTime =
+        taosGetIntervalStartTimestamp(stime, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit, tinfo.precision);
+  
+    taosResetFillInfo(pFillInfo, revisedSTime);
+  }
 
   pLocalReducer->discard = true;
   pLocalReducer->discardData->num = 0;
@@ -915,13 +916,12 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
     if (pQueryInfo->limit.limit >= 0 && pRes->numOfClauseTotal > pQueryInfo->limit.limit) {
       /* impose the limitation of output rows on the final result */
       int32_t prevSize = pFinalDataPage->num;
-      int32_t overFlow = pRes->numOfClauseTotal - pQueryInfo->limit.limit;
-
-      assert(overFlow < pRes->numOfRows);
+      int32_t overflow = pRes->numOfClauseTotal - pQueryInfo->limit.limit;
+      assert(overflow < pRes->numOfRows);
 
       pRes->numOfClauseTotal = pQueryInfo->limit.limit;
-      pRes->numOfRows -= overFlow;
-      pFinalDataPage->num -= overFlow;
+      pRes->numOfRows -= overflow;
+      pFinalDataPage->num -= overflow;
 
       tColModelCompact(pLocalReducer->resColModel, pFinalDataPage, prevSize);
 
@@ -988,13 +988,13 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
 
   if (pRes->numOfRows > 0) {
     if (pQueryInfo->limit.limit >= 0 && pRes->numOfClauseTotal > pQueryInfo->limit.limit) {
-      int32_t overFlow = pRes->numOfClauseTotal - pQueryInfo->limit.limit;
-      pRes->numOfRows -= overFlow;
+      int32_t overflow = pRes->numOfClauseTotal - pQueryInfo->limit.limit;
+      pRes->numOfRows -= overflow;
 
       assert(pRes->numOfRows >= 0);
 
       pRes->numOfClauseTotal = pQueryInfo->limit.limit;
-      pFinalDataPage->num -= overFlow;
+      pFinalDataPage->num -= overflow;
 
       /* set remain data to be discarded, and reset the interpolation information */
       savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pFillInfo);

src/client/src/tscServer.c

@@ -571,7 +571,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid, dfltKey));
 
     pQueryMsg->numOfTables = htonl(1);  // set the number of tables
-
     pMsg += sizeof(STableIdInfo);
   } else {
     int32_t index = pTableMetaInfo->vgroupIndex;
@@ -601,8 +600,8 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     }
   }
   
-  tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
-      pTableMeta->uid);
+  tscTrace("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
+      pTableMeta->sid, pTableMeta->uid);
   
   return pMsg;
 }
@@ -1213,8 +1212,13 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   int8_t type = pInfo->pCreateTableInfo->type;
   if (type == TSQL_CREATE_TABLE_FROM_STABLE) {  // create by using super table, tags value
-    memcpy(pMsg, &pInfo->pCreateTableInfo->usingInfo.tagdata, sizeof(STagData));
-    pMsg += sizeof(STagData);
+    STagData* pTag = &pInfo->pCreateTableInfo->usingInfo.tagdata;
+    *(int32_t*)pMsg = htonl(pTag->dataLen);
+    pMsg += sizeof(int32_t);
+    memcpy(pMsg, pTag->name, sizeof(pTag->name));
+    pMsg += sizeof(pTag->name);
+    memcpy(pMsg, pTag->data, pTag->dataLen);
+    pMsg += pTag->dataLen;
   } else {  // create (super) table
     pSchema = (SSchema *)pCreateTableMsg->schema;
 
@@ -1281,9 +1285,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name);
   pAlterTableMsg->type = htons(pAlterInfo->type);
 
-  pAlterTableMsg->numOfCols = tscNumOfFields(pQueryInfo);
-  memcpy(pAlterTableMsg->tagVal, pAlterInfo->tagData.data, TSDB_MAX_TAGS_LEN);
-
+  pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo));
   SSchema *pSchema = pAlterTableMsg->schema;
   for (int i = 0; i < pAlterTableMsg->numOfCols; ++i) {
     TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
@@ -1295,6 +1297,9 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   }
 
   pMsg = (char *)pSchema;
+  pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen);
+  memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen);
+  pMsg += pAlterInfo->tagData.dataLen;
 
   msgLen = pMsg - (char*)pAlterTableMsg;
   pCmd->payloadLen = msgLen;
@@ -1863,6 +1868,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
   }
 
   free(pTableMeta);
+  tscTrace("%p recv table meta: %"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->uid, pTableMeta->sid, pTableMetaInfo->name);
   
   return TSDB_CODE_SUCCESS;
 }

src/client/src/tscStream.c

@@ -165,7 +165,7 @@ static void tscSetTimestampForRes(SSqlStream *pStream, SSqlObj *pSql) {
   if (timestamp != actualTimestamp) {
     // reset the timestamp of each agg point by using start time of each interval
     *((int64_t *)pRes->data) = actualTimestamp;
-    tscWarn("%p stream:%p, timestamp of points is:%" PRId64 ", reset to %" PRId64 "", pSql, pStream, timestamp, actualTimestamp);
+    tscWarn("%p stream:%p, timestamp of points is:%" PRId64 ", reset to %" PRId64, pSql, pStream, timestamp, actualTimestamp);
   }
 }
 
@@ -287,10 +287,10 @@ static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer)
       return;
     }
   
-    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64 "", pStream->pSql, pStream,
+    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql, pStream,
              now + timer, timer, delay, pStream->stime, etime);
   } else {
-    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64 "", pStream->pSql, pStream,
+    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql, pStream,
              pStream->stime, timer, delay, pStream->stime - pStream->interval, pStream->stime - 1);
   }
 
@@ -380,7 +380,7 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   
   if (pQueryInfo->intervalTime < minIntervalTime) {
-    tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64 "", pSql, pStream,
+    tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64, pSql, pStream,
             pQueryInfo->intervalTime, minIntervalTime);
     pQueryInfo->intervalTime = minIntervalTime;
   }
@@ -397,14 +397,14 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
   if (pQueryInfo->slidingTime == -1) {
     pQueryInfo->slidingTime = pQueryInfo->intervalTime;
   } else if (pQueryInfo->slidingTime < minSlidingTime) {
-    tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64 "", pSql, pStream,
+    tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64, pSql, pStream,
         pQueryInfo->slidingTime, minSlidingTime);
 
     pQueryInfo->slidingTime = minSlidingTime;
   }
 
   if (pQueryInfo->slidingTime > pQueryInfo->intervalTime) {
-    tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64 "", pSql, pStream,
+    tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64, pSql, pStream,
             pQueryInfo->slidingTime, pQueryInfo->intervalTime);
 
     pQueryInfo->slidingTime = pQueryInfo->intervalTime;
@@ -433,11 +433,11 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in
   } else {             // timewindow based aggregation stream
     if (stime == 0) {  // no data in meter till now
       stime = ((int64_t)taosGetTimestamp(pStream->precision) / pStream->interval) * pStream->interval;
-      tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64 "", pSql, pStream, stime);
+      tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64, pSql, pStream, stime);
     } else {
       int64_t newStime = (stime / pStream->interval) * pStream->interval;
       if (newStime != stime) {
-        tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64 "", pSql, pStream, stime, newStime);
+        tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql, pStream, stime, newStime);
         stime = newStime;
       }
     }

src/common/inc/tdataformat.h

@@ -106,9 +106,28 @@ typedef void *SDataRow;
 SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
 void     tdFreeDataRow(SDataRow row);
 void     tdInitDataRow(SDataRow row, STSchema *pSchema);
-int      tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset);
 SDataRow tdDataRowDup(SDataRow row);
 
+static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) {
+  ASSERT(value != NULL);
+  int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
+  char *  ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));
+
+  switch (type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
+      memcpy(ptr, value, varDataTLen(value));
+      dataRowLen(row) += varDataTLen(value);
+      break;
+    default:
+      memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
+      break;
+  }
+
+  return 0;
+}
+
 // NOTE: offset here including the header size
 static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
   switch (type) {

src/common/inc/tglobal.h

@@ -103,6 +103,7 @@ extern int32_t tsOfflineThreshold;
 extern int32_t tsMgmtEqualVnodeNum;
 
 extern int32_t tsEnableHttpModule;
+extern int32_t tsEnableMqttModule;
 extern int32_t tsEnableMonitorModule;
 
 extern int32_t tsRestRowLimit;
@@ -147,6 +148,7 @@ extern int32_t jniDebugFlag;
 extern int32_t tmrDebugFlag;
 extern int32_t sdbDebugFlag;
 extern int32_t httpDebugFlag;
+extern int32_t mqttDebugFlag;
 extern int32_t monitorDebugFlag;
 extern int32_t uDebugFlag;
 extern int32_t rpcDebugFlag;

src/common/src/tdataformat.c

@@ -158,32 +158,6 @@ void tdFreeDataRow(SDataRow row) {
   if (row) free(row);
 }
 
-/**
- * Append a column value to the data row
- * @param type: column type
- * @param bytes: column bytes
- * @param offset: offset in the data row tuple, not including the data row header
- */
-int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) {
-  ASSERT(value != NULL);
-  int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
-  char *  ptr = POINTER_SHIFT(row, dataRowLen(row));
-
-  switch (type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
-      memcpy(ptr, value, varDataTLen(value));
-      dataRowLen(row) += varDataTLen(value);
-      break;
-    default:
-      memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
-      break;
-  }
-
-  return 0;
-}
-
 SDataRow tdDataRowDup(SDataRow row) {
   SDataRow trow = malloc(dataRowLen(row));
   if (trow == NULL) return NULL;
@@ -453,27 +427,25 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCol
     TSKEY key1 = (*iter1 >= src1->numOfPoints) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
     TSKEY key2 = (*iter2 >= src2->numOfPoints) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
 
-    if (key1 < key2) {
+    if (key1 <= key2) {
       for (int i = 0; i < src1->numOfCols; i++) {
         ASSERT(target->cols[i].type == src1->cols[i].type);
-        dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfPoints,
+        dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfPoints,
                          target->maxPoints);
       }
 
       target->numOfPoints++;
       (*iter1)++;
-    } else if (key1 > key2) {
+      if (key1 == key2) (*iter2)++;
+    } else {
       for (int i = 0; i < src2->numOfCols; i++) {
         ASSERT(target->cols[i].type == src2->cols[i].type);
-        dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfPoints,
+        dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfPoints,
                          target->maxPoints);
       }
 
       target->numOfPoints++;
       (*iter2)++;
-    } else {
-      // TODO: deal with duplicate keys
-      ASSERT(false);
     }
   }
 }
\ No newline at end of file

src/common/src/tglobal.c

@@ -107,7 +107,6 @@ int32_t tsReplications  = TSDB_DEFAULT_REPLICA_NUM;
 int16_t tsAffectedRowsMod = 0;
 int32_t tsNumOfMPeers = 3;
 int32_t tsMaxShellConns = 2000;
-int32_t tsMaxTables = 100000;
 
 char    tsDefaultDB[TSDB_DB_NAME_LEN] = {0};
 char    tsDefaultUser[64] = "root";
@@ -121,6 +120,7 @@ int32_t tsOfflineThreshold = 86400*100;   // seconds 10days
 int32_t tsMgmtEqualVnodeNum = 4;
 
 int32_t tsEnableHttpModule = 1;
+int32_t tsEnableMqttModule = 0;   // not finished yet, not started it by default
 int32_t tsEnableMonitorModule = 0;
 
 int32_t tsRestRowLimit = 10240;
@@ -135,13 +135,14 @@ int32_t cDebugFlag = 135;
 int32_t jniDebugFlag = 131;
 int32_t odbcDebugFlag = 131;
 int32_t httpDebugFlag = 131;
+int32_t mqttDebugFlag = 131;
 int32_t monitorDebugFlag = 131;
 int32_t qDebugFlag = 131;
 int32_t rpcDebugFlag = 135;
 int32_t uDebugFlag = 131;
 int32_t debugFlag = 131;
 int32_t sDebugFlag = 135;
-int32_t tsdbDebugFlag = 135;
+int32_t tsdbDebugFlag = 131;
 
 // the maximum number of results for projection query on super table that are returned from
 // one virtual node, to order according to timestamp
@@ -213,6 +214,7 @@ void taosSetAllDebugFlag() {
     jniDebugFlag = debugFlag;
     odbcDebugFlag = debugFlag;
     httpDebugFlag = debugFlag;
+    mqttDebugFlag = debugFlag;
     monitorDebugFlag = debugFlag;
     rpcDebugFlag = debugFlag;
     uDebugFlag = debugFlag;
@@ -891,6 +893,17 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  
+  cfg.option = "mqtt";
+  cfg.ptr = &tsEnableMqttModule;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 1;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "monitor";
   cfg.ptr = &tsEnableMonitorModule;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1113,6 +1126,17 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "mqttDebugFlag";
+  cfg.ptr = &mqttDebugFlag;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
+  cfg.minValue = 0;
+  cfg.maxValue = 255;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+
   cfg.option = "monitorDebugFlag";
   cfg.ptr = &monitorDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;

src/common/src/ttypes.c

@@ -304,7 +304,7 @@ tDataTypeDescriptor tDataTypeDesc[11] = {
   {TSDB_DATA_TYPE_FLOAT,     5, FLOAT_BYTES,  "FLOAT",     tsCompressFloat,     tsDecompressFloat,     getStatics_f},
   {TSDB_DATA_TYPE_DOUBLE,    6, DOUBLE_BYTES, "DOUBLE",    tsCompressDouble,    tsDecompressDouble,    getStatics_d},
   {TSDB_DATA_TYPE_BINARY,    6, 0,            "BINARY",    tsCompressString,    tsDecompressString,    NULL},
-  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, NULL},
+  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
   {TSDB_DATA_TYPE_NCHAR,     5, 8,            "NCHAR",     tsCompressString,    tsDecompressString,    NULL},
 };
 

src/connector/python/linux/python2/taos/cursor.py

@@ -34,6 +34,7 @@ class TDengineCursor(object):
         self._block_rows = -1
         self._block_iter = 0
         self._affected_rows = 0
+        self._logfile = ""
 
         if connection is not None:
             self._connection = connection
@@ -83,6 +84,9 @@ class TDengineCursor(object):
         """
         pass
 
+    def log(self, logfile):
+        self._logfile = logfile
+
     def close(self):
         """Close the cursor.
         """
@@ -113,6 +117,11 @@ class TDengineCursor(object):
             pass
 
         res = CTaosInterface.query(self._connection._conn, stmt)
+
+        if (self._logfile):
+            with open(self._logfile, "a") as logfile:
+                logfile.write("%s;\n" % operation)
+
         if res == 0:
             if CTaosInterface.fieldsCount(self._connection._conn) == 0:
                 self._affected_rows += CTaosInterface.affectedRows(

src/connector/python/linux/python3/taos/cursor.py

@@ -36,6 +36,7 @@ class TDengineCursor(object):
         self._block_rows = -1
         self._block_iter = 0
         self._affected_rows = 0
+        self._logfile = ""
 
         if connection is not None:
             self._connection = connection
@@ -85,6 +86,9 @@ class TDengineCursor(object):
         """
         pass
 
+    def log(self, logfile):
+        self._logfile = logfile
+
     def close(self):
         """Close the cursor.
         """
@@ -121,6 +125,10 @@ class TDengineCursor(object):
         res = CTaosInterface.query(self._connection._conn, stmt)
         # print("   << Query ({}) Exec Done".format(localSeqNum))
 
+        if (self._logfile):
+            with open(self._logfile, "a") as logfile:
+                logfile.write("%s;\n" % operation)
+
         if res == 0:
             if CTaosInterface.fieldsCount(self._connection._conn) == 0:
                 self._affected_rows += CTaosInterface.affectedRows(

src/cq/src/cqMain.c

@@ -40,6 +40,7 @@ typedef struct {
   int      num;      // number of continuous streams
   struct SCqObj *pHead;
   void    *dbConn;
+  int      master;
   pthread_mutex_t mutex;
 } SCqContext;
 
@@ -58,6 +59,7 @@ typedef struct SCqObj {
 int cqDebugFlag = 135;
 
 static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row); 
+static void cqCreateStream(SCqContext *pContext, SCqObj *pObj);
 
 void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
   
@@ -69,6 +71,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
   pContext->vgId = pCfg->vgId;
   pContext->cqWrite = pCfg->cqWrite;
   pContext->ahandle = ahandle;
+  tscEmbedded = 1;
 
   pthread_mutex_init(&pContext->mutex, NULL);
 
@@ -84,6 +87,8 @@ void cqClose(void *handle) {
   cqStop(pContext);
 
   // free all resources
+  pthread_mutex_lock(&pContext->mutex);
+
   SCqObj *pObj = pContext->pHead;
   while (pObj) {
     SCqObj *pTemp = pObj;
@@ -91,6 +96,8 @@ void cqClose(void *handle) {
     free(pTemp);
   } 
   
+  pthread_mutex_unlock(&pContext->mutex);
+
   pthread_mutex_destroy(&pContext->mutex);
 
   cTrace("vgId:%d, CQ is closed", pContext->vgId);
@@ -100,28 +107,15 @@ void cqClose(void *handle) {
 void cqStart(void *handle) {
   SCqContext *pContext = handle;
   cTrace("vgId:%d, start all CQs", pContext->vgId);
-  if (pContext->dbConn) return;
+  if (pContext->dbConn || pContext->master) return;
 
   pthread_mutex_lock(&pContext->mutex);
 
-  tscEmbedded = 1;
-  pContext->dbConn = taos_connect("localhost", pContext->user, pContext->pass, NULL, 0);
-  if (pContext->dbConn == NULL) {
-    cError("vgId:%d, failed to connect to TDengine(%s)", pContext->vgId, tstrerror(terrno));
-    pthread_mutex_unlock(&pContext->mutex);
-    return;
-  }
+  pContext->master = 1;
 
   SCqObj *pObj = pContext->pHead;
   while (pObj) {
-    int64_t lastKey = 0;
-    pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, lastKey, pObj, NULL);
-    if (pObj->pStream) {
-      pContext->num++;
-      cTrace("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
-    } else {
-      cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
-    }
+    cqCreateStream(pContext, pObj);
     pObj = pObj->next;
   }
 
@@ -131,10 +125,11 @@ void cqStart(void *handle) {
 void cqStop(void *handle) {
   SCqContext *pContext = handle;
   cTrace("vgId:%d, stop all CQs", pContext->vgId);
-  if (pContext->dbConn == NULL) return;
+  if (pContext->dbConn == NULL || pContext->master == 0) return;
 
   pthread_mutex_lock(&pContext->mutex);
 
+  pContext->master = 0;
   SCqObj *pObj = pContext->pHead;
   while (pObj) {
     if (pObj->pStream) {
@@ -176,16 +171,7 @@ void *cqCreate(void *handle, int tid, char *sqlStr, SSchema *pSchema, int column
   if (pContext->pHead) pContext->pHead->prev = pObj;
   pContext->pHead = pObj;
 
-  if (pContext->dbConn) {
-    int64_t lastKey = 0;
-    pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, lastKey, pObj, NULL);
-    if (pObj->pStream) {
-      pContext->num++;
-      cTrace("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
-    } else {
-      cError("vgId:%d, id:%d CQ:%s, failed to launch", pContext->vgId, pObj->tid, pObj->sqlStr);
-    }
-  }
+  cqCreateStream(pContext, pObj);
 
   pthread_mutex_unlock(&pContext->mutex);
 
@@ -218,6 +204,26 @@ void cqDrop(void *handle) {
   pthread_mutex_lock(&pContext->mutex);
 }
 
+static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {
+
+  if (pContext->dbConn == NULL) {
+    pContext->dbConn = taos_connect("localhost", pContext->user, pContext->pass, NULL, 0);
+    if (pContext->dbConn == NULL) {
+      cError("vgId:%d, failed to connect to TDengine(%s)", pContext->vgId, tstrerror(terrno));
+    }
+    return;
+  }
+
+  int64_t lastKey = 0;
+  pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, lastKey, pObj, NULL);
+  if (pObj->pStream) {
+    pContext->num++;
+    cTrace("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
+  } else {
+    cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
+  }
+}
+
 static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
   SCqObj     *pObj = (SCqObj *)param;
   SCqContext *pContext = pObj->pContext;

src/dnode/CMakeLists.txt

@@ -16,7 +16,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(src SRC)
 
   ADD_EXECUTABLE(taosd ${SRC})
-  TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb twal vnode cJson lz4)
+  TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4)
 
   IF (TD_ACCOUNT)
     TARGET_LINK_LIBRARIES(taosd account)

src/dnode/src/dnodeModule.c

@@ -20,6 +20,7 @@
 #include "trpc.h"
 #include "mnode.h"
 #include "http.h"
+#include "mqtt.h"
 #include "monitor.h"
 #include "dnodeInt.h"
 #include "dnodeModule.h"
@@ -62,6 +63,16 @@ static void dnodeAllocModules() {
     dnodeSetModuleStatus(TSDB_MOD_HTTP);
   }
 
+  tsModule[TSDB_MOD_MQTT].enable = (tsEnableMqttModule == 1);
+  tsModule[TSDB_MOD_MQTT].name = "mqtt";
+  tsModule[TSDB_MOD_MQTT].initFp = mqttInitSystem;
+  tsModule[TSDB_MOD_MQTT].cleanUpFp = mqttCleanUpSystem;
+  tsModule[TSDB_MOD_MQTT].startFp = mqttStartSystem;
+  tsModule[TSDB_MOD_MQTT].stopFp = mqttStopSystem;
+  if (tsEnableMqttModule) {
+    dnodeSetModuleStatus(TSDB_MOD_MQTT);
+  }
+
   tsModule[TSDB_MOD_MONITOR].enable    = (tsEnableMonitorModule == 1);
   tsModule[TSDB_MOD_MONITOR].name      = "monitor";
   tsModule[TSDB_MOD_MONITOR].initFp    = monitorInitSystem;
@@ -114,18 +125,20 @@ void dnodeStartModules() {
 }
 
 void dnodeProcessModuleStatus(uint32_t moduleStatus) {
-  bool enableMgmtModule = moduleStatus & (1 << TSDB_MOD_MGMT);
-  if (!tsModule[TSDB_MOD_MGMT].enable && enableMgmtModule) {
-    dPrint("module status is received, start mgmt module", tsModuleStatus, moduleStatus);
-    tsModule[TSDB_MOD_MGMT].enable = true;
-    dnodeSetModuleStatus(TSDB_MOD_MGMT);
-    (*tsModule[TSDB_MOD_MGMT].startFp)();
-  }
+  for (int32_t module = TSDB_MOD_MGMT; module < TSDB_MOD_HTTP; ++module) {
+    bool enableModule = moduleStatus & (1 << module);
+    if (!tsModule[module].enable && enableModule) {
+      dPrint("module status:%u is received, start %s module", tsModuleStatus, tsModule[module].name);
+      tsModule[module].enable = true;
+      dnodeSetModuleStatus(module);
+      (*tsModule[module].startFp)();
+    }
 
-  if (tsModule[TSDB_MOD_MGMT].enable && !enableMgmtModule) {
-    dPrint("module status is received, stop mgmt module", tsModuleStatus, moduleStatus);
-    tsModule[TSDB_MOD_MGMT].enable = false;
-    dnodeUnSetModuleStatus(TSDB_MOD_MGMT);
-    (*tsModule[TSDB_MOD_MGMT].stopFp)();
+    if (tsModule[module].enable && !enableModule) {
+      dPrint("module status:%u is received, stop %s module", tsModuleStatus, tsModule[module].name);
+      tsModule[module].enable = false;
+      dnodeUnSetModuleStatus(module);
+      (*tsModule[module].stopFp)();
+    }
   }
 }

src/inc/mqtt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MQTT_H
+#define TDENGINE_MQTT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+int32_t mqttGetReqCount();
+int32_t mqttInitSystem();
+int32_t mqttStartSystem();
+void mqttStopSystem();
+void mqttCleanUpSystem();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/inc/taosdef.h

@@ -340,13 +340,14 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_SESSIONS_PER_VNODE (300)
 #define TSDB_SESSIONS_PER_DNODE (TSDB_SESSIONS_PER_VNODE * TSDB_MAX_VNODES)
 
-#define TSDB_MAX_MNODES        5
-#define TSDB_MAX_DNODES        10
-#define TSDB_MAX_ACCOUNTS      10
-#define TSDB_MAX_USERS         20
-#define TSDB_MAX_DBS           100
-#define TSDB_MAX_VGROUPS       1000
-#define TSDB_MAX_SUPER_TABLES  100
+#define TSDB_DEFAULT_MNODES_HASH_SIZE   5
+#define TSDB_DEFAULT_DNODES_HASH_SIZE   10
+#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10
+#define TSDB_DEFAULT_USERS_HASH_SIZE    20
+#define TSDB_DEFAULT_DBS_HASH_SIZE      100
+#define TSDB_DEFAULT_VGROUPS_HASH_SIZE  100
+#define TSDB_DEFAULT_STABLES_HASH_SIZE  100
+#define TSDB_DEFAULT_CTABLES_HASH_SIZE  10000
 
 #define TSDB_PORT_DNODESHELL 0 
 #define TSDB_PORT_DNODEDNODE 5 
@@ -375,6 +376,7 @@ typedef enum {
   TSDB_MOD_MGMT,
   TSDB_MOD_HTTP,
   TSDB_MOD_MONITOR,
+  TSDB_MOD_MQTT,
   TSDB_MOD_MAX
 } EModuleType;
 

src/inc/taoserror.h

@@ -156,6 +156,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY,         0, 405, "server out of m
 TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS,        0, 406, "no disk permissions")
 TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED,             0, 407, "file corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED,           0, 408, "memory corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUCH_FILE_OR_DIR,       0, 409, "no such file or directory")
 
 // client
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION,     0, 451, "invalid client version")

src/inc/taosmsg.h

@@ -269,9 +269,11 @@ typedef struct {
   char    tableId[TSDB_TABLE_ID_LEN + 1];
   char    db[TSDB_DB_NAME_LEN + 1];
   int16_t type; /* operation type   */
-  char    tagVal[TSDB_MAX_BYTES_PER_ROW];
-  int8_t  numOfCols; /* number of schema */
+  int16_t numOfCols; /* number of schema */
+  int32_t tagValLen;
   SSchema schema[];
+  // tagVal is padded after schema
+  // char    tagVal[];
 } SCMAlterTableMsg;
 
 typedef struct {
@@ -647,6 +649,7 @@ typedef struct SMultiTableMeta {
 } SMultiTableMeta;
 
 typedef struct {
+  int32_t dataLen;
   char name[TSDB_TABLE_ID_LEN + 1];
   char data[TSDB_MAX_TAGS_LEN];
 } STagData;
@@ -722,6 +725,8 @@ typedef struct {
 typedef struct {
   uint32_t  queryId;
   uint32_t  streamId;
+  uint32_t  totalDnodes;
+  uint32_t  onlineDnodes;
   int8_t    killConnection;
   SRpcIpSet ipList;
 } SCMHeartBeatRsp;

src/kit/taosdump/taosdump.c

@@ -37,6 +37,8 @@
 #define COMMAND_SIZE 65536
 #define DEFAULT_DUMP_FILE "taosdump.sql"
 
+#define MAX_DBS  100
+
 int  converStringToReadable(char *str, int size, char *buf, int bufsize);
 int  convertNCharToReadable(char *str, int size, char *buf, int bufsize);
 void taosDumpCharset(FILE *fp);
@@ -359,7 +361,7 @@ int main(int argc, char *argv[]) {
 
 void taosFreeDbInfos() {
   if (dbInfos == NULL) return;
-  for (int i = 0; i < TSDB_MAX_DBS; i++) tfree(dbInfos[i]);
+  for (int i = 0; i < MAX_DBS; i++) tfree(dbInfos[i]);
   tfree(dbInfos);
 }
 
@@ -437,7 +439,7 @@ int taosDumpOut(SDumpArguments *arguments) {
     return -1;
   }
 
-  dbInfos = (SDbInfo **)calloc(TSDB_MAX_DBS, sizeof(SDbInfo *));
+  dbInfos = (SDbInfo **)calloc(MAX_DBS, sizeof(SDbInfo *));
   if (dbInfos == NULL) {
     fprintf(stderr, "failed to allocate memory\n");
     goto _exit_failure;
@@ -992,7 +994,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) {
           pstr += sprintf(pstr, "%d", *((int *)row[col]));
           break;
         case TSDB_DATA_TYPE_BIGINT:
-          pstr += sprintf(pstr, "%" PRId64 "", *((int64_t *)row[col]));
+          pstr += sprintf(pstr, "%" PRId64, *((int64_t *)row[col]));
           break;
         case TSDB_DATA_TYPE_FLOAT:
           pstr += sprintf(pstr, "%f", GET_FLOAT_VAL(row[col]));
@@ -1011,7 +1013,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) {
           pstr += sprintf(pstr, "\'%s\'", tbuf);
           break;
         case TSDB_DATA_TYPE_TIMESTAMP:
-          pstr += sprintf(pstr, "%" PRId64 "", *(int64_t *)row[col]);
+          pstr += sprintf(pstr, "%" PRId64, *(int64_t *)row[col]);
           break;
         default:
           break;

src/mnode/inc/mgmtDef.h

@@ -66,7 +66,6 @@ typedef struct SMnodeObj {
   SDnodeObj *pDnode;
 } SMnodeObj;
 
-// todo use dynamic length string
 typedef struct {
   char  *tableId;
   int8_t type;

src/mnode/inc/mgmtDnode.h

@@ -34,6 +34,7 @@ char*   mgmtGetDnodeStatusStr(int32_t dnodeStatus);
 void    mgmtMonitorDnodeModule();
 
 int32_t mgmtGetDnodesNum();
+int32_t mgmtGetOnlinDnodesNum();
 void *  mgmtGetNextDnode(void *pIter, SDnodeObj **pDnode);
 void    mgmtIncDnodeRef(SDnodeObj *pDnode);
 void    mgmtDecDnodeRef(SDnodeObj *pDnode);

src/mnode/inc/mgmtMnode.h

@@ -44,7 +44,7 @@ void    mgmtDecMnodeRef(struct SMnodeObj *pMnode);
 char *  mgmtGetMnodeRoleStr();
 void    mgmtGetMnodeIpSet(SRpcIpSet *ipSet);
 void    mgmtGetMnodeInfos(void *mnodes);
-
+void    mgmtUpdateMnodeIpSet();
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtTable.h

@@ -31,6 +31,7 @@ void *  mgmtGetNextChildTable(void *pIter, SChildTableObj **pTable);
 void *  mgmtGetNextSuperTable(void *pIter, SSuperTableObj **pTable);
 void    mgmtDropAllChildTables(SDbObj *pDropDb);
 void    mgmtDropAllSuperTables(SDbObj *pDropDb);
+void    mgmtDropAllChildTablesInVgroups(SVgObj *pVgroup);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtVgroup.h

@@ -29,6 +29,7 @@ void    mgmtIncVgroupRef(SVgObj *pVgroup);
 void    mgmtDecVgroupRef(SVgObj *pVgroup);
 void    mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg);
 void    mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode);
+void    mgmtUpdateAllDbVgroups(SDbObj *pAlterDb);
 
 void *  mgmtGetNextVgroup(void *pIter, SVgObj **pVgroup);
 void    mgmtUpdateVgroup(SVgObj *pVgroup);

src/mnode/src/mgmtAcct.c

@@ -94,7 +94,7 @@ int32_t mgmtInitAccts() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_ACCOUNT,
     .tableName    = "accounts",
-    .hashSessions = TSDB_MAX_ACCOUNTS,
+    .hashSessions = TSDB_DEFAULT_ACCOUNTS_HASH_SIZE,
     .maxRowSize   = tsAcctUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_STRING,

src/mnode/src/mgmtDb.c

@@ -96,6 +96,7 @@ static int32_t mgmtDbActionUpdate(SSdbOper *pOper) {
     memcpy(pSaved, pDb, pOper->rowSize);
     free(pDb);
   }
+  mgmtUpdateAllDbVgroups(pSaved);
   mgmtDecDbRef(pSaved);
   return TSDB_CODE_SUCCESS;
 }
@@ -127,7 +128,7 @@ int32_t mgmtInitDbs() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_DB,
     .tableName    = "dbs",
-    .hashSessions = TSDB_MAX_DBS,
+    .hashSessions = TSDB_DEFAULT_DBS_HASH_SIZE,
     .maxRowSize   = tsDbUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_STRING,

src/mnode/src/mgmtDnode.c

@@ -74,7 +74,6 @@ static int32_t mgmtDnodeActionDelete(SSdbOper *pOper) {
   SDnodeObj *pDnode = pOper->pObj;
  
 #ifndef _SYNC 
-  //TODO: drop dnode local
   mgmtDropAllDnodeVgroups(pDnode);
 #endif  
   mgmtDropMnodeLocal(pDnode->dnodeId);
@@ -130,7 +129,7 @@ int32_t mgmtInitDnodes() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_DNODE,
     .tableName    = "dnodes",
-    .hashSessions = TSDB_MAX_DNODES,
+    .hashSessions = TSDB_DEFAULT_DNODES_HASH_SIZE,
     .maxRowSize   = tsDnodeUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_AUTO,
@@ -179,6 +178,23 @@ int32_t mgmtGetDnodesNum() {
   return sdbGetNumOfRows(tsDnodeSdb);
 }
 
+int32_t mgmtGetOnlinDnodesNum(char *ep) {
+  SDnodeObj *pDnode = NULL;
+  void *     pIter = NULL;
+  int32_t    onlineDnodes = 0;
+
+  while (1) {
+    pIter = mgmtGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (pDnode->status != TAOS_DN_STATUS_OFFLINE) onlineDnodes++;
+    mgmtDecDnodeRef(pDnode);
+  }
+
+  sdbFreeIter(pIter);
+
+  return onlineDnodes;
+}
+
 void *mgmtGetDnode(int32_t dnodeId) {
   return sdbGetRow(tsDnodeSdb, &dnodeId);
 }
@@ -397,7 +413,6 @@ static int32_t mgmtCreateDnode(char *ep) {
   return code;
 }
 
-//TODO drop others tables
 int32_t mgmtDropDnode(SDnodeObj *pDnode) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
@@ -410,7 +425,7 @@ int32_t mgmtDropDnode(SDnodeObj *pDnode) {
     code = TSDB_CODE_SDB_ERROR;
   }
 
-  mLPrint("dnode:%d is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
+  mLPrint("dnode:%d, is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
   return code;
 }
 

src/mnode/src/mgmtMnode.c

@@ -36,6 +36,25 @@ static int32_t tsMnodeUpdateSize = 0;
 static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
+static SRpcIpSet tsMnodeRpcIpSet;
+static SDMMnodeInfos tsMnodeInfos;
+
+#if defined(LINUX)
+  static pthread_rwlock_t        tsMnodeLock;
+  #define mgmtMnodeWrLock()      pthread_rwlock_wrlock(&tsMnodeLock)
+  #define mgmtMnodeRdLock()      pthread_rwlock_rdlock(&tsMnodeLock)
+  #define mgmtMnodeUnLock()      pthread_rwlock_unlock(&tsMnodeLock)
+  #define mgmtMnodeInitLock()    pthread_rwlock_init(&tsMnodeLock, NULL)
+  #define mgmtMnodeDestroyLock() pthread_rwlock_destroy(&tsMnodeLock)
+#else
+  static pthread_mutex_t         tsMnodeLock;
+  #define mgmtMnodeWrLock()      pthread_mutex_lock(&tsMnodeLock)
+  #define mgmtMnodeRdLock()      pthread_mutex_lock(&tsMnodeLock)
+  #define mgmtMnodeUnLock()      pthread_mutex_unlock(&tsMnodeLock)
+  #define mgmtMnodeInitLock()    pthread_mutex_init(&tsMnodeLock, NULL)
+  #define mgmtMnodeDestroyLock() pthread_mutex_destroy(&tsMnodeLock)
+#endif
+
 static int32_t mgmtMnodeActionDestroy(SSdbOper *pOper) {
   tfree(pOper->pObj);
   return TSDB_CODE_SUCCESS;
@@ -102,17 +121,22 @@ static int32_t mgmtMnodeActionRestored() {
     }
     sdbFreeIter(pIter);
   }
+
+  mgmtUpdateMnodeIpSet();
+
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t mgmtInitMnodes() {
+  mgmtMnodeInitLock();
+
   SMnodeObj tObj;
   tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
 
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_MNODE,
     .tableName    = "mnodes",
-    .hashSessions = TSDB_MAX_MNODES,
+    .hashSessions = TSDB_DEFAULT_MNODES_HASH_SIZE,
     .maxRowSize   = tsMnodeUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_INT,
@@ -140,6 +164,7 @@ int32_t mgmtInitMnodes() {
 
 void mgmtCleanupMnodes() {
   sdbCloseTable(tsMnodeSdb);
+  mgmtMnodeDestroyLock();
 }
 
 int32_t mgmtGetMnodesNum() { 
@@ -177,50 +202,57 @@ char *mgmtGetMnodeRoleStr(int32_t role) {
   }
 }
 
-void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
-  void *pIter = NULL;
-  while (1) {
-    SMnodeObj *pMnode = NULL;
-    pIter = mgmtGetNextMnode(pIter, &pMnode);
-    if (pMnode == NULL) break;
-
-    strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn);
-    ipSet->port[ipSet->numOfIps] = htons(pMnode->pDnode->dnodePort);
+void mgmtUpdateMnodeIpSet() {
+  SRpcIpSet *ipSet = &tsMnodeRpcIpSet;
+  SDMMnodeInfos *mnodes = &tsMnodeInfos;
 
-    if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
-      ipSet->inUse = ipSet->numOfIps;
-    }
+  mPrint("update mnodes ipset, numOfIps:%d ", mgmtGetMnodesNum());
 
-    ipSet->numOfIps++;
-    
-    mgmtDecMnodeRef(pMnode);
-  }
-  sdbFreeIter(pIter);
-}
+  mgmtMnodeWrLock();
 
-void mgmtGetMnodeInfos(void *param) {
-  SDMMnodeInfos *mnodes = param;
-  mnodes->inUse = 0;
-  
   int32_t index = 0;
-  void *pIter = NULL;
+  void *  pIter = NULL;
   while (1) {
     SMnodeObj *pMnode = NULL;
     pIter = mgmtGetNextMnode(pIter, &pMnode);
     if (pMnode == NULL) break;
 
+    strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn);
+    ipSet->port[ipSet->numOfIps] = htons(pMnode->pDnode->dnodePort);
+
     mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
     strcpy(mnodes->nodeInfos[index].nodeEp, pMnode->pDnode->dnodeEp);
+
     if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
+      ipSet->inUse = ipSet->numOfIps;
       mnodes->inUse = index;
     }
 
+    mPrint("mnode:%d, ep:%s %s", index, pMnode->pDnode->dnodeEp, pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : "");
+
+    ipSet->numOfIps++;
     index++;
+    
     mgmtDecMnodeRef(pMnode);
   }
-  sdbFreeIter(pIter);
 
   mnodes->nodeNum = index;
+
+  sdbFreeIter(pIter);
+
+  mgmtMnodeUnLock();
+}
+
+void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
+  mgmtMnodeRdLock();
+  *ipSet = tsMnodeRpcIpSet;
+  mgmtMnodeUnLock();
+}
+
+void mgmtGetMnodeInfos(void *mnodeInfos) {
+  mgmtMnodeRdLock();
+  *(SDMMnodeInfos *)mnodeInfos = tsMnodeInfos;
+  mgmtMnodeUnLock();
 }
 
 int32_t mgmtAddMnode(int32_t dnodeId) {
@@ -240,6 +272,8 @@ int32_t mgmtAddMnode(int32_t dnodeId) {
     code = TSDB_CODE_SDB_ERROR;
   }
 
+  mgmtUpdateMnodeIpSet();
+
   return code;
 }
 
@@ -250,6 +284,8 @@ void mgmtDropMnodeLocal(int32_t dnodeId) {
     sdbDeleteRow(&oper);
     mgmtDecMnodeRef(pMnode);
   }
+
+  mgmtUpdateMnodeIpSet();
 }
 
 int32_t mgmtDropMnode(int32_t dnodeId) {
@@ -270,6 +306,9 @@ int32_t mgmtDropMnode(int32_t dnodeId) {
   }
 
   sdbDecRef(tsMnodeSdb, pMnode);
+
+  mgmtUpdateMnodeIpSet();
+
   return code;
 }
 

src/mnode/src/mgmtSdb.c

@@ -196,6 +196,8 @@ void sdbUpdateMnodeRoles() {
       mgmtDecMnodeRef(pMnode);
     }
   }
+
+  mgmtUpdateMnodeIpSet();
 }
 
 static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion) {
@@ -352,7 +354,7 @@ void sdbIncRef(void *handle, void *pObj) {
   int32_t *  pRefCount = (int32_t *)(pObj + pTable->refCountPos);
   atomic_add_fetch_32(pRefCount, 1);
   if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) {
-    sdbTrace("table:%s, add ref to record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
+    sdbTrace("add ref to table:%s record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
   }
 }
 
@@ -363,7 +365,7 @@ void sdbDecRef(void *handle, void *pObj) {
   int32_t *  pRefCount = (int32_t *)(pObj + pTable->refCountPos);
   int32_t    refCount = atomic_sub_fetch_32(pRefCount, 1);
   if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) {
-    sdbTrace("table:%s, def ref of record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
+    sdbTrace("def ref of table:%s record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
   }
 
   int8_t *updateEnd = pObj + pTable->refCountPos - 1;
@@ -442,8 +444,8 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
 
   pthread_mutex_unlock(&pTable->mutex);
 
-  sdbTrace("table:%s, insert record:%s to hash, numOfRows:%d version:%" PRIu64, pTable->tableName,
-           sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
+  sdbTrace("table:%s, insert record:%s to hash, rowSize:%d vnumOfRows:%d version:%" PRIu64, pTable->tableName,
+           sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion());
 
   (*pTable->insertFp)(pOper);
   return TSDB_CODE_SUCCESS;

src/mnode/src/mgmtShell.c

@@ -119,7 +119,7 @@ static void mgmtDoDealyedAddToShellQueue(void *param, void *tmrId) {
 
 void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg) {
   void *unUsed = NULL;
-  taosTmrReset(mgmtDoDealyedAddToShellQueue, 1000, queuedMsg, tsMgmtTmr, &unUsed);
+  taosTmrReset(mgmtDoDealyedAddToShellQueue, 300, queuedMsg, tsMgmtTmr, &unUsed);
 }
 
 void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
@@ -325,6 +325,8 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
     return;
   }
 
+  pHBRsp->onlineDnodes = htonl(mgmtGetOnlinDnodesNum());
+  pHBRsp->totalDnodes = htonl(mgmtGetDnodesNum());
   mgmtGetMnodeIpSet(&pHBRsp->ipList);
   
   /*

src/mnode/src/mgmtTable.c

@@ -340,7 +340,7 @@ static int32_t mgmtInitChildTables() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_CTABLE,
     .tableName    = "ctables",
-    .hashSessions = tsMaxTables,
+    .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE,
     .maxRowSize   = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN + TSDB_CQ_SQL_SIZE,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_VAR_STRING,
@@ -507,7 +507,7 @@ static int32_t mgmtInitSuperTables() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_STABLE,
     .tableName    = "stables",
-    .hashSessions = TSDB_MAX_SUPER_TABLES,
+    .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE,
     .maxRowSize   = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_VAR_STRING,
@@ -1305,7 +1305,7 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
         if (pDnode == NULL) break;
 
         strncpy(pVgroupInfo->vgroups[vgSize].ipAddr[vn].fqdn, pDnode->dnodeFqdn, tListLen(pDnode->dnodeFqdn));
-        pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(tsDnodeShellPort);
+        pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(pDnode->dnodePort);
 
         pVgroupInfo->vgroups[vgSize].numOfIps++;
       }
@@ -1334,13 +1334,13 @@ static void mgmtProcessDropSuperTableRsp(SRpcMsg *rpcMsg) {
 }
 
 static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) {
-  char *  pTagData = NULL;
+  STagData *  pTagData = NULL;
   int32_t tagDataLen = 0;
   int32_t totalCols = 0;
   int32_t contLen = 0;
   if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) {
-    pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1;
-    tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1;
+    pTagData = (STagData*)pMsg->schema;
+    tagDataLen = ntohl(pTagData->dataLen);
     totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags;
     contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen + pTable->sqlLen;
   } else {
@@ -1393,7 +1393,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb
   }
 
   if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) {
-    memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen);
+    memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData->data, tagDataLen);
     memcpy(pCreate->data + totalCols * sizeof(SSchema) + tagDataLen, pTable->sql, pTable->sqlLen);
   }
 
@@ -1420,10 +1420,10 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj
   pTable->vgId         = pVgroup->vgId;
     
   if (pTable->info.type == TSDB_CHILD_TABLE) {
-    char *pTagData = (char *) pCreate->schema;  // it is a tag key
-    SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData);
+    STagData *pTagData = (STagData *) pCreate->schema;  // it is a tag key
+    SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData->name);
     if (pSuperTable == NULL) {
-      mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData);
+      mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData->name);
       free(pTable);
       terrno = TSDB_CODE_INVALID_TABLE;
       return NULL;
@@ -1538,7 +1538,7 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
   SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
   SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
   newMsg->ahandle = pMsg->pTable;
-  newMsg->maxRetry = 5;
+  newMsg->maxRetry = 10;
   SRpcMsg rpcMsg = {
       .handle  = newMsg,
       .pCont   = pMDCreate,
@@ -1742,7 +1742,9 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
 
 static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) {
   SCMTableInfoMsg *pInfo = pMsg->pCont;
-  int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData);
+  STagData* pTag = (STagData*)pInfo->tags;
+
+  int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + ntohl(pTag->dataLen);
   SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
   if (pCreateMsg == NULL) {
     mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId);
@@ -1756,14 +1758,9 @@ static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) {
   pCreateMsg->getMeta = 1;
   pCreateMsg->contLen = htonl(contLen);
 
-  contLen = sizeof(STagData);
-  if (contLen > pMsg->contLen - sizeof(SCMTableInfoMsg)) {
-    contLen = pMsg->contLen - sizeof(SCMTableInfoMsg);
-  }
-  memcpy(pCreateMsg->schema, pInfo->tags, contLen);
+  memcpy(pCreateMsg->schema, pInfo->tags, contLen - sizeof(SCMCreateTableMsg));
 
   SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
-  pMsg->pCont = newMsg->pCont;
   newMsg->msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE;
   newMsg->pCont = pCreateMsg;
 
@@ -1790,6 +1787,34 @@ static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) {
   rpcSendResponse(&rpcRsp);
 }
 
+void mgmtDropAllChildTablesInVgroups(SVgObj *pVgroup) {
+  void *  pIter = NULL;
+  int32_t numOfTables = 0;
+  SChildTableObj *pTable = NULL;
+
+  mPrint("vgId:%d, all child tables will be dropped from sdb", pVgroup->vgId);
+
+  while (1) {
+    pIter = mgmtGetNextChildTable(pIter, &pTable);
+    if (pTable == NULL) break;
+
+    if (pTable->vgId == pVgroup->vgId) {
+      SSdbOper oper = {
+        .type = SDB_OPER_LOCAL,
+        .table = tsChildTableSdb,
+        .pObj = pTable,
+      };
+      sdbDeleteRow(&oper);
+      numOfTables++;
+    }
+    mgmtDecTableRef(pTable);
+  }
+
+  sdbFreeIter(pIter);
+
+  mPrint("vgId:%d, all child tables is dropped from sdb", pVgroup->vgId);
+}
+
 void mgmtDropAllChildTables(SDbObj *pDropDb) {
   void *  pIter = NULL;
   int32_t numOfTables = 0;
@@ -2201,6 +2226,8 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
   }
 
   pAlter->type = htons(pAlter->type);
+  pAlter->numOfCols = htons(pAlter->numOfCols);
+  pAlter->tagValLen = htonl(pAlter->tagValLen);
 
   if (pAlter->numOfCols > 2) {
     mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols);
@@ -2232,7 +2259,8 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
     mTrace("table:%s, start to alter ctable", pAlter->tableId);
     SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
     if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) {
-      code = mgmtModifyChildTableTagValue(pTable, pAlter->schema[0].name, pAlter->tagVal);
+      char *tagVal = (char*)(pAlter->schema + pAlter->numOfCols);
+      code = mgmtModifyChildTableTagValue(pTable, pAlter->schema[0].name, tagVal);
     } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) {
       code = mgmtAddNormalTableColumn(pMsg->pDb, pTable, pAlter->schema, 1);
     } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) {

src/mnode/src/mgmtUser.c

@@ -117,7 +117,7 @@ int32_t mgmtInitUsers() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_USER,
     .tableName    = "users",
-    .hashSessions = TSDB_MAX_USERS,
+    .hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE,
     .maxRowSize   = tsUserUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_STRING,

src/mnode/src/mgmtVgroup.c

@@ -121,6 +121,20 @@ static int32_t mgmtVgroupActionDelete(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
+static void mgmtVgroupUpdateIdPool(SVgObj *pVgroup) {
+  int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool);
+  SDbObj *pDb = pVgroup->pDb;
+  if (pDb != NULL) {
+    if (pDb->cfg.maxTables != oldTables) {
+      mPrint("vgId:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
+      taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
+      int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
+      pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
+      memset(pVgroup->tableList + oldTables, 0, (pDb->cfg.maxTables - oldTables) * sizeof(SChildTableObj **));
+    }
+  }
+}
+
 static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
   SVgObj *pNew = pOper->pObj;
   SVgObj *pVgroup = mgmtGetVgroup(pNew->vgId);
@@ -146,20 +160,11 @@ static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
     }
   }
 
-  int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool);
-  SDbObj *pDb = pVgroup->pDb;
-  if (pDb != NULL) {
-    if (pDb->cfg.maxTables != oldTables) {
-      mPrint("vgId:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
-      taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
-      int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
-      pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
-    }
-  }
+  mgmtVgroupUpdateIdPool(pVgroup);
 
   mgmtDecVgroupRef(pVgroup);
 
-  mTrace("vgId:%d, is updated, numOfVnode:%d tables:%d", pVgroup->vgId, pVgroup->numOfVnodes, pDb == NULL ? 0 : pDb->cfg.maxTables);
+  mTrace("vgId:%d, is updated, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -196,7 +201,7 @@ int32_t mgmtInitVgroups() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_VGROUP,
     .tableName    = "vgroups",
-    .hashSessions = TSDB_MAX_VGROUPS,
+    .hashSessions = TSDB_DEFAULT_VGROUPS_HASH_SIZE,
     .maxRowSize   = tsVgUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_AUTO,
@@ -742,11 +747,14 @@ void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
   SVgObj *pVgroup = NULL;
   int32_t numOfVgroups = 0;
 
+  mPrint("dnode:%d, all vgroups will be dropped from sdb", pDropDnode->dnodeId);
+
   while (1) {
     pIter = mgmtGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;
 
     if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) {
+      mgmtDropAllChildTablesInVgroups(pVgroup);
       SSdbOper oper = {
         .type = SDB_OPER_LOCAL,
         .table = tsVgroupSdb,
@@ -754,12 +762,35 @@ void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
       };
       sdbDeleteRow(&oper);
       numOfVgroups++;
-      continue;
     }
     mgmtDecVgroupRef(pVgroup);
   }
 
   sdbFreeIter(pIter);
+
+  mPrint("dnode:%d, all vgroups is dropped from sdb", pDropDnode->dnodeId);
+}
+
+void mgmtUpdateAllDbVgroups(SDbObj *pAlterDb) {
+  void *  pIter = NULL;
+  SVgObj *pVgroup = NULL;
+
+  mPrint("db:%s, all vgroups will be update in sdb", pAlterDb->name);
+
+  while (1) {
+    pIter = mgmtGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    if (pVgroup->pDb == pAlterDb) {
+      mgmtVgroupUpdateIdPool(pVgroup);
+    }
+
+    mgmtDecVgroupRef(pVgroup);
+  }
+
+  sdbFreeIter(pIter);
+
+  mPrint("db:%s, all vgroups is updated in sdb", pAlterDb->name);
 }
 
 void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {

src/os/linux/src/linuxPlatform.c

@@ -148,6 +148,10 @@ static void taosDeleteTimer(void *tharg) {
   timer_delete(*pTimer);
 }
 
+static pthread_t timerThread;
+static timer_t         timerId;
+static volatile bool stopTimer = false;
+
 void *taosProcessAlarmSignal(void *tharg) {
   // Block the signal
   sigset_t sigset;
@@ -156,7 +160,6 @@ void *taosProcessAlarmSignal(void *tharg) {
   sigprocmask(SIG_BLOCK, &sigset, NULL);
   void (*callback)(int) = tharg;
 
-  static timer_t         timerId;
   struct sigevent sevent = {{0}};
 
   #ifdef _ALPINE
@@ -187,7 +190,7 @@ void *taosProcessAlarmSignal(void *tharg) {
   }
 
   int signo;
-  while (1) {
+  while (!stopTimer) {
     if (sigwait(&sigset, &signo)) {
       uError("Failed to wait signal: number %d", signo);
       continue;
@@ -202,7 +205,6 @@ void *taosProcessAlarmSignal(void *tharg) {
   return NULL;
 }
 
-static pthread_t timerThread;
 
 int taosInitTimer(void (*callback)(int), int ms) {
   pthread_attr_t tattr;
@@ -217,7 +219,7 @@ int taosInitTimer(void (*callback)(int), int ms) {
 }
 
 void taosUninitTimer() {
-  pthread_cancel(timerThread);
+  stopTimer = true;
   pthread_join(timerThread, NULL);
 }
 

src/plugins/CMakeLists.txt

@@ -3,3 +3,4 @@ PROJECT(TDengine)
 
 ADD_SUBDIRECTORY(monitor)
 ADD_SUBDIRECTORY(http)
+ADD_SUBDIRECTORY(mqtt)

src/plugins/http/src/httpServer.c

@@ -267,8 +267,10 @@ static void httpStopThread(HttpThread* pThread) {
   struct epoll_event event = { .events = EPOLLIN };
   eventfd_t fd = eventfd(1, 0);
   if (fd == -1) {
+    httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno));
     pthread_cancel(pThread->thread);
   } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
+    httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno));
     pthread_cancel(pThread->thread);
   }
 

src/plugins/mqtt/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
+  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
+  INCLUDE_DIRECTORIES(inc)
+  AUX_SOURCE_DIRECTORY(src SRC)
+  ADD_LIBRARY(mqtt ${SRC})
+  TARGET_LINK_LIBRARIES(mqtt taos_static z)
+
+  IF (TD_ADMIN)
+    TARGET_LINK_LIBRARIES(mqtt admin)
+  ENDIF ()
+ENDIF ()

src/plugins/mqtt/inc/mqttLog.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MQTT_LOG_H
+#define TDENGINE_MQTT_LOG_H
+
+#include "tlog.h"
+
+extern int32_t mqttDebugFlag;
+
+#define mqttError(...)                       \
+  if (mqttDebugFlag & DEBUG_ERROR) {         \
+    taosPrintLog("ERROR MQT ", 255, __VA_ARGS__); \
+  }
+#define mqttWarn(...)                                  \
+  if ( mqttDebugFlag & DEBUG_WARN) {                    \
+    taosPrintLog("WARN  MQT ",  mqttDebugFlag, __VA_ARGS__); \
+  }
+#define  mqttTrace(...)                           \
+  if ( mqttDebugFlag & DEBUG_TRACE) {             \
+    taosPrintLog("MQT ",  mqttDebugFlag, __VA_ARGS__); \
+  }
+#define  mqttDump(...)                                        \
+  if ( mqttDebugFlag & DEBUG_TRACE) {                         \
+    taosPrintLongString("MQT ",  mqttDebugFlag, __VA_ARGS__); \
+  }
+#define  mqttPrint(...) \
+  { taosPrintLog("MQT ", 255, __VA_ARGS__); }
+
+#endif

src/plugins/mqtt/inc/mqttSystem.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MQTT_SYSTEM_H
+#define TDENGINE_MQTT_SYSTEM_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+int32_t mqttGetReqCount();
+int32_t mqttInitSystem();
+int32_t mqttStartSystem();
+void    mqttStopSystem();
+void    mqttCleanUpSystem();
+ 
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/plugins/mqtt/src/mqttSystem.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mqttSystem.h"
+#include "mqtt.h"
+#include "mqttLog.h"
+#include "os.h"
+#include "taos.h"
+#include "tglobal.h"
+#include "tsocket.h"
+#include "ttimer.h"
+
+int32_t mqttGetReqCount() { return 0; }
+int     mqttInitSystem() {
+  mqttPrint("mqttInitSystem");
+  return 0;
+}
+
+int mqttStartSystem() {
+  mqttPrint("mqttStartSystem");
+  return 0;
+}
+
+void mqttStopSystem() {
+	mqttPrint("mqttStopSystem");
+}
+
+void mqttCleanUpSystem() { 
+	mqttPrint("mqttCleanUpSystem");
+}

src/query/src/qExecutor.c

@@ -97,7 +97,9 @@ typedef struct {
   STSCursor   cur;
 } SQueryStatusInfo;
 
+#define CLEAR_QUERY_STATUS(q, st)   ((q)->status &= (~(st)))
 static void setQueryStatus(SQuery *pQuery, int8_t status);
+
 static bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; }
 
 // todo move to utility
@@ -278,6 +280,26 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) {
   return maxOutput;
 }
 
+/*
+ * the value of number of result needs to be update due to offset value upated.
+ */
+void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) {
+  SQuery *pQuery = pRuntimeEnv->pQuery;
+  
+  for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
+    SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
+    
+    int16_t functionId = pRuntimeEnv->pCtx[j].functionId;
+    if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ ||
+        functionId == TSDB_FUNC_TS_DUMMY) {
+      continue;
+    }
+    
+    assert(pResInfo->numOfRes > numOfRes);
+    pResInfo->numOfRes = numOfRes;
+  }
+}
+
 static int32_t getGroupResultId(int32_t groupIndex) {
   int32_t base = 200000;
   return base + (groupIndex * 10000);
@@ -354,9 +376,7 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) {
 
 bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.functionId == TSDB_FUNC_TS_COMP; }
 
-static bool limitResults(SQInfo *pQInfo) {
-  SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
-
+static bool limitResults(SQuery *pQuery) {
   if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) {
     pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total;
     assert(pQuery->rec.rows > 0);
@@ -626,6 +646,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
     int32_t i = 0;
     int64_t skey = TSKEY_INITIAL_VAL;
 
+    // TODO opt performance: get the closed time window here
     for (i = 0; i < pWindowResInfo->size; ++i) {
       SWindowResult *pResult = &pWindowResInfo->pResult[i];
       if (pResult->status.closed) {
@@ -1303,6 +1324,10 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
     if (numOfRes >= pQuery->rec.threshold) {
       setQueryStatus(pQuery, QUERY_RESBUF_FULL);
     }
+    
+    if ((pQuery->limit.limit >= 0) && numOfRes >= (pQuery->limit.limit + pQuery->limit.offset)) {
+      setQueryStatus(pQuery, QUERY_COMPLETED);
+    }
   }
 
   return numOfRes;
@@ -2408,6 +2433,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
 
           char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage));
           if (tmp == NULL) {  // todo handle the oom
+            assert(0);
           } else {
             pQuery->sdata[i] = (tFilePage *)tmp;
           }
@@ -2421,7 +2447,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
           }
         }
   
-        qTrace("QInfo: %p realloc output buffer, new size: %d rows, old:%d, remain:%d", GET_QINFO_ADDR(pRuntimeEnv),
+        qTrace("QInfo:%p realloc output buffer, new size: %d rows, old:%d, remain:%d", GET_QINFO_ADDR(pRuntimeEnv),
                newSize, pRec->capacity, newSize - pRec->rows);
         
         pRec->capacity = newSize;
@@ -2434,11 +2460,11 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
     pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1;
     int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock);
 
-    qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv),
+    qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, numOfRes:%d", GET_QINFO_ADDR(pRuntimeEnv),
            blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes);
 
-    // save last access position
-    if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) {
+    // while the output buffer is full or limit/offset is applied, query may be paused here
+    if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL | QUERY_COMPLETED)) {
       break;
     }
   }
@@ -3004,11 +3030,13 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *
   
   // order has change already!
   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
-  if (!QUERY_IS_ASC_QUERY(pQuery)) {
-    assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step);
-  } else {
-    assert(pTableQueryInfo->win.ekey <= pTableQueryInfo->lastKey + step);
-  }
+  
+  // TODO validate the assertion
+//  if (!QUERY_IS_ASC_QUERY(pQuery)) {
+//    assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step);
+//  } else {
+//    assert(pTableQueryInfo->win.ekey <= pTableQueryInfo->lastKey + step);
+//  }
   
   pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step;
   
@@ -3087,7 +3115,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) {
 void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-    SWITCH_ORDER(pRuntimeEnv->pCtx[i] .order);
+    SWITCH_ORDER(pRuntimeEnv->pCtx[i].order);
   }
 }
 
@@ -3173,30 +3201,38 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) {
   }
 
   if (pQuery->rec.rows <= pQuery->limit.offset) {
+    qTrace("QInfo:%p skip rows:%d, new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows,
+        pQuery->limit.offset - pQuery->rec.rows);
+    
     pQuery->limit.offset -= pQuery->rec.rows;
     pQuery->rec.rows = 0;
 
     resetCtxOutputBuf(pRuntimeEnv);
 
-    // clear the buffer is full flag if exists
-    pQuery->status &= (~QUERY_RESBUF_FULL);
+    // clear the buffer full flag if exists
+    CLEAR_QUERY_STATUS(pQuery, QUERY_RESBUF_FULL);
   } else {
-    int32_t numOfSkip = (int32_t) pQuery->limit.offset;
+    int64_t numOfSkip = pQuery->limit.offset;
     pQuery->rec.rows -= numOfSkip;
-
+    pQuery->limit.offset = 0;
+  
+    qTrace("QInfo:%p skip row:%"PRId64", new offset:%d, numOfRows remain:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), numOfSkip,
+           0, pQuery->rec.rows);
+    
     for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
       int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
       int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
       
-      memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->rec.rows * bytes);
-      pRuntimeEnv->pCtx[i].aOutputBuf += bytes * numOfSkip;
+      memmove(pQuery->sdata[i]->data, (char*) pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->rec.rows * bytes);
+      pRuntimeEnv->pCtx[i].aOutputBuf = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes;
 
       if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) {
-        pRuntimeEnv->pCtx[i].ptsOutputBuf += TSDB_KEYSIZE * numOfSkip;
+        pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
       }
     }
-
-    pQuery->limit.offset = 0;
+  
+    
+    updateNumOfResult(pRuntimeEnv, pQuery->rec.rows);
   }
 }
 
@@ -3205,7 +3241,7 @@ void setQueryStatus(SQuery *pQuery, int8_t status) {
     pQuery->status = status;
   } else {
     // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first
-    pQuery->status &= (~QUERY_NOT_COMPLETED);
+    CLEAR_QUERY_STATUS(pQuery, QUERY_NOT_COMPLETED);
     pQuery->status |= status;
   }
 }
@@ -3957,7 +3993,7 @@ static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBloc
 
   int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock);
 
-  qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv),
+  qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, numOfRes:%d", GET_QINFO_ADDR(pRuntimeEnv),
          pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes);
 }
 
@@ -3986,7 +4022,7 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
       pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey;
       pTableQueryInfo->lastKey += step;
 
-      qTrace("QInfo:%p skip rows:%d, offset:%" PRId64 "", GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows,
+      qTrace("QInfo:%p skip rows:%d, offset:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows,
              pQuery->limit.offset);
     } else {  // find the appropriated start position in current block
       updateOffsetVal(pRuntimeEnv, &blockInfo);
@@ -4075,7 +4111,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
           int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock);
           pRuntimeEnv->windowResInfo.curIndex = index;  // restore the window index
           
-          qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d",
+          qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, numOfRes:%d",
                  GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes);
           return true;
         } else { // do nothing
@@ -4350,10 +4386,11 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) {
     
     SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
 
-    TSKEY nextKey = blockInfo.window.skey;
     if (!isIntervalQuery(pQuery)) {
-      setExecutionContext(pQInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIdx, nextKey);
+      int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1;
+      setExecutionContext(pQInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIdx, blockInfo.window.ekey + step);
     } else {  // interval query
+      TSKEY nextKey = blockInfo.window.skey;
       setIntervalQueryRange(pQInfo, nextKey);
       int32_t ret = setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo);
 
@@ -4532,8 +4569,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
         continue;
       }
 
-      //      SPointInterpoSupporter pointInterpSupporter = {0};
-
       // TODO handle the limit offset problem
       if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) {
         //        skipBlocks(pRuntimeEnv);
@@ -4544,12 +4579,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       }
 
       scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
-
-      pQuery->rec.rows = getNumOfResult(pRuntimeEnv);
       skipResults(pRuntimeEnv);
 
       // the limitation of output result is reached, set the query completed
-      if (limitResults(pQInfo)) {
+      if (limitResults(pQuery)) {
         pQInfo->tableIndex = pQInfo->groupInfo.numOfTables;
         break;
       }
@@ -4578,18 +4611,15 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
           break;
         }
 
-      } else {  // forward query range
-        pQuery->window.skey = pQuery->current->lastKey;
-
+      } else {
         // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter
         if (pQuery->rec.rows == 0) {
           assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL));
           continue;
         } else {
-          //          pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey;
-          //          // buffer is full, wait for the next round to retrieve data from current meter
-          //          assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL));
-          //          break;
+          // buffer is full, wait for the next round to retrieve data from current meter
+          assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL));
+          break;
         }
       }
     }
@@ -4633,10 +4663,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
     copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult);
   }
 
-  pQuery->rec.total += pQuery->rec.rows;
-
   qTrace(
-      "QInfo %p, numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%"PRId64", offset:%" PRId64,
+      "QInfo %p numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%"PRId64", offset:%" PRId64,
       pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total,
       pQuery->limit.offset);
 }
@@ -4809,7 +4837,7 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
   pQuery->rec.rows = getNumOfResult(pRuntimeEnv);
 
   skipResults(pRuntimeEnv);
-  limitResults(pQInfo);
+  limitResults(pQuery);
 }
 
 static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
@@ -4857,7 +4885,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
     resetCtxOutputBuf(pRuntimeEnv);
   }
 
-  limitResults(pQInfo);
+  limitResults(pQuery);
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) {
     qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo,
         pQuery->current->lastKey, pQuery->window.ekey);
@@ -4935,7 +4963,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
 
     // the offset is handled at prepare stage if no interpolation involved
     if (pQuery->fillType == TSDB_FILL_NONE || pQuery->rec.rows == 0) {
-      limitResults(pQInfo);
+      limitResults(pQuery);
       break;
     } else {
       TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime,
@@ -4947,7 +4975,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
 
       qTrace("QInfo: %p fill results completed, final:%d", pQInfo, pQuery->rec.rows);
       if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
-        limitResults(pQInfo);
+        limitResults(pQuery);
         break;
       }
 
@@ -4982,7 +5010,7 @@ static void tableQueryImpl(SQInfo *pQInfo) {
   
     qTrace("QInfo: %p fill results completed, final:%d", pQInfo, pQuery->rec.rows);
     if (pQuery->rec.rows > 0) {
-      limitResults(pQInfo);
+      limitResults(pQuery);
     }
     
     qTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total);
@@ -5106,7 +5134,7 @@ bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SC
 
 static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) {
   if (pQueryMsg->intervalTime < 0) {
-    qError("qmsg:%p illegal value of interval time %" PRId64 "", pQueryMsg, pQueryMsg->intervalTime);
+    qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->intervalTime);
     return false;
   }
 
@@ -6013,6 +6041,8 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
     isSTableQuery = TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
     
     STableIdInfo *id = taosArrayGet(pTableIdList, 0);
+    qTrace("qmsg:%p query table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
+    
     if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) {
       goto _over;
     }

src/query/src/qtokenizer.c

@@ -282,11 +282,7 @@ int tSQLKeywordCode(const char* z, int n) {
   }
 
   SKeyword** pKey = (SKeyword**)taosHashGet(KeywordHashTable, key, n);
-  if (pKey != NULL) {
-    return (*pKey)->type;
-  } else {
-    return TK_ID;
-  }
+  return (pKey != NULL)? (*pKey)->type:TK_ID;
 }
 
 /*
@@ -594,31 +590,28 @@ SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn
   while (1) {
     *i += t0.n;
 
-    bool hasComma = false;
-    while ((str[*i] == ' ' || str[*i] == '\n' || str[*i] == '\r' || str[*i] == '\t' || str[*i] == '\f')
-        || str[*i] == ',') {
-      if (str[*i] == ',') {
-        if (false == hasComma) {
-          hasComma = true;
-        } else {  // comma only allowed once
-          t0.n = 0;
-          return t0;
-        }
+    int32_t numOfComma = 0;
+    char t = str[*i];
+    while (t == ' ' || t == '\n' || t == '\r' || t == '\t' || t == '\f' || t == ',') {
+      if (t == ',' && (++numOfComma > 1)) {  // comma only allowed once
+        t0.n = 0;
+        return t0;
       }
-      (*i)++;
+    
+      t = str[++(*i)];
     }
 
     t0.n = tSQLGetToken(&str[*i], &t0.type);
 
-    bool ignoreFlag = false;
+    bool ignore = false;
     for (uint32_t k = 0; k < numOfIgnoreToken; k++) {
       if (t0.type == ignoreTokenTypes[k]) {
-        ignoreFlag = true;
+        ignore = true;
         break;
       }
     }
 
-    if (!ignoreFlag) {
+    if (!ignore) {
       break;
     }
   }
@@ -662,114 +655,4 @@ SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn
   return t0;
 }
 
-FORCE_INLINE bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); }
-
+bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); }
\ No newline at end of file
-FORCE_INLINE bool isNumber(const SSQLToken* pToken) {
-  return (pToken->type == TK_INTEGER || pToken->type == TK_FLOAT || pToken->type == TK_HEX || pToken->type == TK_BIN);
-}
-
-int32_t isValidNumber(const SSQLToken* pToken) {
-  const char* z = pToken->z;
-  int32_t type = TK_ILLEGAL;
-
-  int32_t i = 0;
-  for(; i < pToken->n; ++i) {
-    switch (z[i]) {
-      case '+':
-      case '-': {
-        break;
-      }
-      case '.': {
-          /*
-           * handle the the float number with out integer part
-           * .123
-           * .123e4
-           */
-          if (!isdigit(z[i+1])) {
-            return TK_ILLEGAL;
-          }
-
-          for (i += 2; isdigit(z[i]); i++) {
-          }
-
-          if ((z[i] == 'e' || z[i] == 'E') &&
-              (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) {
-            i += 2;
-            while (isdigit(z[i])) {
-              i++;
-            }
-          }
-
-          type = TK_FLOAT;
-          goto _end;
-      }
-
-      case '0': {
-        char next = z[i + 1];
-        if (next == 'b') { // bin number
-          type = TK_BIN;
-          for (i += 2; (z[i] == '0' || z[i] == '1'); ++i) {
-          }
-
-          goto _end;
-        } else if (next == 'x') {  //hex number
-          type = TK_HEX;
-          for (i += 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) {
-          }
-
-          goto _end;
-        }
-      }
-      case '1':
-      case '2':
-      case '3':
-      case '4':
-      case '5':
-      case '6':
-      case '7':
-      case '8':
-      case '9': {
-        type = TK_INTEGER;
-        for (; isdigit(z[i]); i++) {
-        }
-
-        int32_t seg = 0;
-        while (z[i] == '.' && isdigit(z[i + 1])) {
-          i += 2;
-
-          while (isdigit(z[i])) {
-            i++;
-          }
-
-          seg++;
-          type = TK_FLOAT;
-        }
-
-        if (seg > 1) {
-          return TK_ILLEGAL;
-        }
-
-        if ((z[i] == 'e' || z[i] == 'E') &&
-            (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) {
-          i += 2;
-          while (isdigit(z[i])) {
-            i++;
-          }
-
-          type = TK_FLOAT;
-        }
-
-        goto _end;
-      }
-      default:
-        return TK_ILLEGAL;
-    }
-  }
-
-  _end:
-  if (i < pToken->n) {
-    return TK_ILLEGAL;
-  } else {
-    return type;
-  }
-}
\ No newline at end of file

src/rpc/src/rpcMain.c

@@ -867,9 +867,8 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
   // underlying UDP layer does not know it is server or client
   pRecv->connType = pRecv->connType | pRpc->connType;  
 
-  if (pRecv->ip==0 && pConn) {
-    rpcProcessBrokenLink(pConn); 
-    tfree(pRecv->msg);
+  if (pRecv->ip == 0 && pConn) {
+    rpcProcessBrokenLink(pConn);
     return NULL;
   }
 
@@ -894,7 +893,7 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
     }
   }
 
-  if (code) tfree(pRecv->msg); // parsing failed, msg shall be freed
+  if (code) rpcFreeMsg(pRecv->msg); // parsing failed, msg shall be freed
   return pConn;
 }
 

src/rpc/src/rpcTcp.c

@@ -147,8 +147,10 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
   struct epoll_event event = { .events = EPOLLIN };
   eventfd_t fd = eventfd(1, 0);
   if (fd == -1) {
+    tError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThreadObj->label, strerror(errno));
     pthread_cancel(pThreadObj->thread);
   } else if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
+    tError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThreadObj->label, strerror(errno));
     pthread_cancel(pThreadObj->thread);
   }
 
@@ -213,7 +215,6 @@ static void* taosAcceptTcpConnection(void *arg) {
       continue;
     }
 
-    tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(caddr.sin_addr), caddr.sin_port);
     taosKeepTcpAlive(connFd);
 
     // pick up the thread to handle this connection
@@ -227,7 +228,8 @@ static void* taosAcceptTcpConnection(void *arg) {
               inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds);
     } else {
       close(connFd);
-      tError("%s failed to malloc FdObj(%s)", pServerObj->label, strerror(errno));
+      tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno),
+             inet_ntoa(caddr.sin_addr), caddr.sin_port);
     }  
 
     // pick up next thread for next connection
@@ -339,7 +341,9 @@ static void taosReportBrokenLink(SFdObj *pFdObj) {
     recvInfo.chandle = NULL;
     recvInfo.connType = RPC_CONN_TCP;
     (*(pThreadObj->processData))(&recvInfo);
-  } 
+  } else {
+    taosFreeFdObj(pFdObj);
+  }
 }
 
 #define maxEvents 10
@@ -350,7 +354,7 @@ static void *taosProcessTcpData(void *param) {
   struct epoll_event events[maxEvents];
   SRecvInfo          recvInfo;
   SRpcHead           rpcHead;
-
+ 
   while (1) {
     int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1);
     if (pThreadObj->stop) {
@@ -464,7 +468,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
 
   pFdObj->signature = NULL;
   epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
-  close(pFdObj->fd);
+  taosCloseTcpSocket(pFdObj->fd);
 
   pThreadObj->numOfFds--;
 

src/rpc/test/rserver.c

@@ -127,6 +127,8 @@ int main(int argc, char *argv[]) {
   SRpcInit rpcInit;
   char     dataName[20] = "server.data";
 
+  taosBlockSIGPIPE();
+
   memset(&rpcInit, 0, sizeof(rpcInit));
   rpcInit.localPort    = 7000;
   rpcInit.label        = "SER";

src/tsdb/inc/tsdbMain.h

@@ -495,11 +495,12 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo);
 int  tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError);
 
 // --------- For read operations
-int tsdbLoadCompIdx(SRWHelper *pHelper, void *target);
-int tsdbLoadCompInfo(SRWHelper *pHelper, void *target);
-int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target);
-int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds);
-int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target);
+int  tsdbLoadCompIdx(SRWHelper *pHelper, void *target);
+int  tsdbLoadCompInfo(SRWHelper *pHelper, void *target);
+int  tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target);
+int  tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds);
+int  tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target);
+void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols);
 
 // --------- For write operations
 int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols);

src/tsdb/src/tsdbCache.c

@@ -162,6 +162,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
     pCache->totalCacheBlocks = totalBlocks;
     tsdbAdjustCacheBlocks(pCache);
   }
+  pRepo->config.totalBlocks = totalBlocks;
 
   tsdbUnLockRepo((TsdbRepoT *)pRepo);
   tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks);

src/tsdb/src/tsdbMain.c

@@ -37,7 +37,8 @@ static TSKEY   tsdbNextIterKey(SSkipListIterator *pIter);
 static int     tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey);
 static void    tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression);
 static void    tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep);
-static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables);
+static void    tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables);
+static int32_t tsdbSaveConfig(STsdbRepo *pRepo);
 
 #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid]
 #define TSDB_GET_TABLE_BY_NAME(pRepo, name)
@@ -319,10 +320,25 @@ int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg) {
   ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock);
   ASSERT(pRCfg->precision == pCfg->precision);
 
-  if (pRCfg->compression != pCfg->compression) tsdbAlterCompression(pRepo, pCfg->compression);
-  if (pRCfg->keep != pCfg->keep) tsdbAlterKeep(pRepo, pCfg->keep);
-  if (pRCfg->totalBlocks != pCfg->totalBlocks) tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks);
-  if (pRCfg->maxTables != pCfg->maxTables) tsdbAlterMaxTables(pRepo, pCfg->maxTables);
+  bool configChanged = false;
+  if (pRCfg->compression != pCfg->compression) {
+    configChanged = true;
+    tsdbAlterCompression(pRepo, pCfg->compression);
+  }
+  if (pRCfg->keep != pCfg->keep) {
+    configChanged = true;
+    tsdbAlterKeep(pRepo, pCfg->keep);
+  }
+  if (pRCfg->totalBlocks != pCfg->totalBlocks) {
+    configChanged = true;
+    tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks);
+  }
+  if (pRCfg->maxTables != pCfg->maxTables) {
+    configChanged = true;
+    tsdbAlterMaxTables(pRepo, pCfg->maxTables);
+  }
+
+  if (configChanged) tsdbSaveConfig(pRepo);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1134,8 +1150,10 @@ static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) {
 
   int maxFiles = keep / pCfg->maxTables + 3;
   if (pRepo->config.keep > keep) {
+    pRepo->config.keep = keep;
     pRepo->tsdbFileH->maxFGroups = maxFiles;
   } else {
+    pRepo->config.keep = keep;
     pRepo->tsdbFileH->fGroup = realloc(pRepo->tsdbFileH->fGroup, sizeof(SFileGroup));
     if (pRepo->tsdbFileH->fGroup == NULL) {
       // TODO: deal with the error
@@ -1155,6 +1173,8 @@ static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) {
 
   pMeta->maxTables = maxTables;
   pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *));
+  memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables-oldMaxTables));
+  pRepo->config.maxTables = maxTables;
 
   tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables);
 }

src/tsdb/src/tsdbMeta.c

@@ -16,7 +16,7 @@ static int     tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx)
 static int     tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbEstimateTableEncodeSize(STable *pTable);
-static int     tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable);
+static int     tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable, bool rmFromIdx);
 
 /**
  * Encode a TSDB table object as a binary content
@@ -127,7 +127,7 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
   if (pTable->type == TSDB_SUPER_TABLE) {
     STColumn* pColSchema = schemaColAt(pTable->tagSchema, 0);
     pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes,
-                                    1, 0, 0, getTagIndexKey);
+                                    1, 0, 1, getTagIndexKey);
   }
 
   tsdbAddTableToMeta(pMeta, pTable, false);
@@ -289,6 +289,13 @@ int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
 
   if (tsdbCheckTableCfg(pCfg) < 0) return -1;
 
+  STable *pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid);
+  if (pTable != NULL) {
+    tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name),
+              pTable->tableId.tid, pTable->tableId.uid);
+    return TSDB_CODE_TABLE_ALREADY_EXIST;
+  }
+
   STable *super = NULL;
   int newSuper = 0;
 
@@ -316,7 +323,7 @@ int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
       // index the first tag column
       STColumn* pColSchema = schemaColAt(super->tagSchema, 0);
       super->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes,
-          1, 0, 0, getTagIndexKey);  // Allow duplicate key, no lock
+          1, 0, 1, getTagIndexKey);  // Allow duplicate key, no lock
 
       if (super->pIndex == NULL) {
         tdFreeSchema(super->schema);
@@ -411,7 +418,7 @@ int tsdbDropTable(TsdbRepoT *repo, STableId tableId) {
 
   tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name),
             tableId.tid, tableId.uid);
-  if (tsdbRemoveTableFromMeta(pMeta, pTable) < 0) return -1;
+  if (tsdbRemoveTableFromMeta(pMeta, pTable, true) < 0) return -1;
 
   return 0;
 
@@ -440,6 +447,7 @@ static int tsdbFreeTable(STable *pTable) {
 
   // Free content
   if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) {
+    tdFreeSchema(pTable->tagSchema);
     tSkipListDestroy(pTable->pIndex);
   }
 
@@ -500,7 +508,7 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) {
   return 0;
 }
 
-static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
+static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable, bool rmFromIdx) {
   if (pTable->type == TSDB_SUPER_TABLE) {
     SSkipListIterator  *pIter = tSkipListCreateIter(pTable->pIndex);
     while (tSkipListIterNext(pIter)) {
@@ -509,7 +517,7 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
 
       ASSERT(tTable != NULL && tTable->type == TSDB_CHILD_TABLE);
 
-      tsdbRemoveTableFromMeta(pMeta, tTable);
+      tsdbRemoveTableFromMeta(pMeta, tTable, false);
     }
 
     tSkipListDestroyIter(pIter);
@@ -525,7 +533,7 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
     }
   } else {
     pMeta->tables[pTable->tableId.tid] = NULL;
-    if (pTable->type == TSDB_CHILD_TABLE) {
+    if (pTable->type == TSDB_CHILD_TABLE && rmFromIdx) {
       tsdbRemoveTableFromIndex(pMeta, pTable);
     }
 

src/tsdb/src/tsdbRWHelper.c

@@ -409,7 +409,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
     if (pIdx->offset > 0) {
       pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
       if (pIdx->offset < 0) return -1;
-      ASSERT(pIdx->offset >= tsizeof(pHelper->pCompIdx));
+      ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
 
       if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) return -1;
     }
@@ -489,6 +489,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
       }
 
       ASSERT(((char *)ptr - (char *)pHelper->pBuffer) == (pFile->info.len - sizeof(TSCKSUM)));
+      if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
     }
 
   }
@@ -542,6 +543,34 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) {
   return 0;
 }
 
+void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols) {
+  SCompData *pCompData = pHelper->pCompData;
+
+  for (int i = 0, j = 0; i < numOfCols;) {
+    if (j >= pCompData->numOfCols) {
+      pStatis[i].numOfNull = -1;
+      i++;
+      continue;
+    }
+
+    if (pStatis[i].colId == pCompData->cols[j].colId) {
+      pStatis[i].sum = pCompData->cols[j].sum;
+      pStatis[i].max = pCompData->cols[j].max;
+      pStatis[i].min = pCompData->cols[j].min;
+      pStatis[i].maxIndex = pCompData->cols[j].maxIndex;
+      pStatis[i].minIndex = pCompData->cols[j].minIndex;
+      pStatis[i].numOfNull = pCompData->cols[j].numOfNull;
+      i++;
+      j++;
+    } else if (pStatis[i].colId < pCompData->cols[j].colId) {
+      pStatis[i].numOfNull = -1;
+      i++;
+    } else {
+      j++;
+    }
+  }
+}
+
 static int comparColIdCompCol(const void *arg1, const void *arg2) {
   return (*(int16_t *)arg1) - ((SCompCol *)arg2)->colId;
 }
@@ -747,7 +776,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
 
     pCompCol->colId = pDataCol->colId;
     pCompCol->type = pDataCol->type;
-    if (tDataTypeDesc[pDataCol->type].getStatisFunc) {
+    if (tDataTypeDesc[pDataCol->type].getStatisFunc && ncol != 0) {
       (*tDataTypeDesc[pDataCol->type].getStatisFunc)(
           (TSKEY *)(pDataCols->cols[0].pData), pDataCol->pData, rowsToWrite, &(pCompCol->min), &(pCompCol->max),
           &(pCompCol->sum), &(pCompCol->minIndex), &(pCompCol->maxIndex), &(pCompCol->numOfNull));

src/util/inc/tstoken.h

@@ -21,6 +21,8 @@ extern "C" {
 #endif
 
 #include "os.h"
+#include "tutil.h"
+#include "ttokendef.h"
 
 #define TK_SPACE      200
 #define TK_COMMENT    201
@@ -31,7 +33,7 @@ extern "C" {
 #define TK_FILE       206
 #define TK_QUESTION   207   // denoting the placeholder of "?",when invoking statement bind query
 
-#define TSQL_TBNAME "TBNAME"
+#define TSQL_TBNAME   "TBNAME"
 #define TSQL_TBNAME_L "tbname"
 
 // used to denote the minimum unite in sql parsing
@@ -74,14 +76,117 @@ bool isKeyWord(const char *z, int32_t len);
  * @param pToken
  * @return
  */
-bool isNumber(const SSQLToken *pToken);
+#define isNumber(tk) \
+((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN)
+
 
 /**
  * check if it is a token or not
  * @param pToken
- * @return       token type, if it is not a number, TK_ILLEGAL will return
+ * @return        token type, if it is not a number, TK_ILLEGAL will return
  */
-int32_t isValidNumber(const SSQLToken* pToken);
+static FORCE_INLINE int32_t isValidNumber(const SSQLToken* pToken) {
+  const char* z = pToken->z;
+  int32_t type = TK_ILLEGAL;
+
+  int32_t i = 0;
+  for(; i < pToken->n; ++i) {
+    switch (z[i]) {
+      case '+':
+      case '-': {
+        break;
+      }
+
+      case '.': {
+        /*
+         * handle the the float number with out integer part
+         * .123
+         * .123e4
+         */
+        if (!isdigit(z[i+1])) {
+          return TK_ILLEGAL;
+        }
+
+        for (i += 2; isdigit(z[i]); i++) {
+        }
+
+        if ((z[i] == 'e' || z[i] == 'E') &&
+            (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) {
+          i += 2;
+          while (isdigit(z[i])) {
+            i++;
+          }
+        }
+
+        type = TK_FLOAT;
+        goto _end;
+      }
+
+      case '0': {
+        char next = z[i + 1];
+        if (next == 'b') { // bin number
+          type = TK_BIN;
+          for (i += 2; (z[i] == '0' || z[i] == '1'); ++i) {
+          }
+
+          goto _end;
+        } else if (next == 'x') {  //hex number
+          type = TK_HEX;
+          for (i += 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) {
+          }
+
+          goto _end;
+        }
+      }
+      case '1':
+      case '2':
+      case '3':
+      case '4':
+      case '5':
+      case '6':
+      case '7':
+      case '8':
+      case '9': {
+        type = TK_INTEGER;
+        for (; isdigit(z[i]); i++) {
+        }
+
+        int32_t seg = 0;
+        while (z[i] == '.' && isdigit(z[i + 1])) {
+          i += 2;
+
+          while (isdigit(z[i])) {
+            i++;
+          }
+
+          seg++;
+          type = TK_FLOAT;
+        }
+
+        if (seg > 1) {
+          return TK_ILLEGAL;
+        }
+
+        if ((z[i] == 'e' || z[i] == 'E') &&
+            (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) {
+          i += 2;
+          while (isdigit(z[i])) {
+            i++;
+          }
+
+          type = TK_FLOAT;
+        }
+
+        goto _end;
+      }
+      default:
+        return TK_ILLEGAL;
+    }
+  }
+
+  _end:
+  return (i < pToken->n)? TK_ILLEGAL:type;
+}
 
 #ifdef __cplusplus
 }

src/util/src/hash.c

@@ -102,7 +102,32 @@ static void doUpdateHashTable(SHashObj *pHashObj, SHashNode *pNode);
  * @param hashVal   hash value by hash function
  * @return
  */
-static SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal);
+FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal) {
+  uint32_t hash = (*pHashObj->hashFp)(key, keyLen);
+  
+  int32_t     slot = HASH_INDEX(hash, pHashObj->capacity);
+  SHashEntry *pEntry = pHashObj->hashList[slot];
+  
+  SHashNode *pNode = pEntry->next;
+  while (pNode) {
+    if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
+      break;
+    }
+    
+    pNode = pNode->next;
+  }
+  
+  if (pNode) {
+    assert(HASH_INDEX(pNode->hashVal, pHashObj->capacity) == slot);
+  }
+  
+  // return the calculated hash value, to avoid calculating it again in other functions
+  if (hashVal != NULL) {
+    *hashVal = hash;
+  }
+  
+  return pNode;
+}
 
 /**
  * Resize the hash list if the threshold is reached
@@ -438,33 +463,6 @@ void doUpdateHashTable(SHashObj *pHashObj, SHashNode *pNode) {
   }
 }
 
-SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal) {
-  uint32_t hash = (*pHashObj->hashFp)(key, keyLen);
-  
-  int32_t     slot = HASH_INDEX(hash, pHashObj->capacity);
-  SHashEntry *pEntry = pHashObj->hashList[slot];
-  
-  SHashNode *pNode = pEntry->next;
-  while (pNode) {
-    if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
-      break;
-    }
-    
-    pNode = pNode->next;
-  }
-  
-  if (pNode) {
-    assert(HASH_INDEX(pNode->hashVal, pHashObj->capacity) == slot);
-  }
-  
-  // return the calculated hash value, to avoid calculating it again in other functions
-  if (hashVal != NULL) {
-    *hashVal = hash;
-  }
-  
-  return pNode;
-}
-
 void taosHashTableResize(SHashObj *pHashObj) {
   if (pHashObj->size < pHashObj->capacity * HASH_DEFAULT_LOAD_FACTOR) {
     return;

src/util/src/tcompare.c

@@ -216,14 +216,14 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c
   return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
 }
 
-static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
+static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
   
-  const char* pattern = pRight;
-  const char* str = pLeft;
-  
-  int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo);
+  char pattern[128] = {0};
+  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
+  assert(varDataLen(pRight) < 128);
   
+  int32_t ret = patternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft), &pInfo);
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 
@@ -232,14 +232,14 @@ static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) {
   return taosArraySearchString(arr, pLeft) == NULL ? 0 : 1;
 }
 
-static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
+static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
   
-  const wchar_t* pattern = pRight;
-  const wchar_t* str = pLeft;
-  
-  int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo);
+  wchar_t pattern[128] = {0};
+  memcpy(pattern, varDataVal(pRight), varDataLen(pRight)/TSDB_NCHAR_SIZE);
+  assert(varDataLen(pRight) < 128);
   
+  int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo);
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 

src/util/src/thashutil.c

@@ -10,7 +10,7 @@
 #include "hashfunc.h"
 #include "tutil.h"
 
-#define ROTL32(x, r) ((x) << (r) | (x) >> (32 - (r)))
+#define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r)))
 
 #define FMIX32(h)      \
   do {                 \
@@ -20,12 +20,12 @@
     (h) *= 0xc2b2ae35; \
     (h) ^= (h) >> 16;  \
   } while (0)
-
-static void MurmurHash3_32_s(const void *key, int len, uint32_t seed, void *out) {
+  
+uint32_t MurmurHash3_32(const char *key, uint32_t len) {
   const uint8_t *data = (const uint8_t *)key;
-  const int      nblocks = len / 4;
+  const int nblocks = len >> 2u;
 
-  uint32_t h1 = seed;
+  uint32_t h1 = 0x12345678;
 
   const uint32_t c1 = 0xcc9e2d51;
   const uint32_t c2 = 0x1b873593;
@@ -36,11 +36,11 @@ static void MurmurHash3_32_s(const void *key, int len, uint32_t seed, void *out)
     uint32_t k1 = blocks[i];
 
     k1 *= c1;
-    k1 = ROTL32(k1, 15);
+    k1 = ROTL32(k1, 15u);
     k1 *= c2;
 
     h1 ^= k1;
-    h1 = ROTL32(h1, 13);
+    h1 = ROTL32(h1, 13u);
     h1 = h1 * 5 + 0xe6546b64;
   }
 
@@ -48,7 +48,7 @@ static void MurmurHash3_32_s(const void *key, int len, uint32_t seed, void *out)
 
   uint32_t k1 = 0;
 
-  switch (len & 3) {
+  switch (len & 3u) {
     case 3:
       k1 ^= tail[2] << 16;
     case 2:
@@ -56,7 +56,7 @@ static void MurmurHash3_32_s(const void *key, int len, uint32_t seed, void *out)
     case 1:
       k1 ^= tail[0];
       k1 *= c1;
-      k1 = ROTL32(k1, 15);
+      k1 = ROTL32(k1, 15u);
       k1 *= c2;
       h1 ^= k1;
   };
@@ -65,16 +65,7 @@ static void MurmurHash3_32_s(const void *key, int len, uint32_t seed, void *out)
 
   FMIX32(h1);
 
-  *(uint32_t *)out = h1;
-}
-
-uint32_t MurmurHash3_32(const char *key, uint32_t len) {
-  const int32_t hashSeed = 0x12345678;
-
-  uint32_t val = 0;
-  MurmurHash3_32_s(key, len, hashSeed, &val);
-
-  return val;
+  return h1;
 }
 
 uint32_t taosIntHash_32(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint32_t *)key; }

src/vnode/src/vnodeMain.c

@@ -73,13 +73,18 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
     return TSDB_CODE_SUCCESS;
   }
 
+  mkdir(tsVnodeDir, 0755);
+
   char rootDir[TSDB_FILENAME_LEN] = {0};
   sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId);
   if (mkdir(rootDir, 0755) != 0) {
+    vPrint("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), rootDir);
     if (errno == EACCES) {
       return TSDB_CODE_NO_DISK_PERMISSIONS;
     } else if (errno == ENOSPC) {
       return TSDB_CODE_SERV_NO_DISKSPACE;
+    } else if (errno == ENOENT) {
+      return TSDB_CODE_NOT_SUCH_FILE_OR_DIR;
     } else if (errno == EEXIST) {
     } else {
       return TSDB_CODE_VG_INIT_FAILED;
@@ -239,6 +244,10 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   syncInfo.notifyFileSynced = vnodeNotifyFileSynced;
   pVnode->sync = syncStart(&syncInfo);
 
+#ifndef _SYNC
+  pVnode->role = TAOS_SYNC_ROLE_MASTER;
+#endif
+
   // start continuous query
   if (pVnode->role == TAOS_SYNC_ROLE_MASTER) 
     cqStart(pVnode->cq);
@@ -429,7 +438,7 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) {
 
 static void vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) {
   SVnodeObj *pVnode = ahandle;
-  vTrace("vgId:%d, data file is synced, fversion:%" PRId64 "", pVnode->vgId, fversion);
+  vTrace("vgId:%d, data file is synced, fversion:%" PRId64, pVnode->vgId, fversion);
 
   pVnode->fversion = fversion;
   pVnode->version = fversion;

tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md

@@ -11,6 +11,19 @@
 4.  pip install src/connector/python/linux/python2 ; pip3 install
     src/connector/python/linux/python3
 
+>   Note: Both Python2 and Python3 are currently supported by the Python test
+>   framework. Since Python2 is no longer officially supported by Python Software
+>   Foundation since January 1, 2020, it is recommended that subsequent test case
+>   development be guaranteed to run correctly on Python3. 
+
+>   For Python2, please consider being compatible if appropriate without 
+>   additional burden.
+>
+>   If you use some new Linux distribution like Ubuntu 20.04 which already do not
+>   include Python2, please do not install Python2-related packages.
+>
+>   <https://nakedsecurity.sophos.com/2020/01/03/python-is-dead-long-live-python/> 
+
 ### How to run Python test suite
 
 1.  cd \<TDengine\>/tests/pytest
@@ -211,13 +224,6 @@ def checkAffectedRows(self, expectAffectedRows):
 
 ...
 
->   Note: Both Python2 and Python3 are currently supported by the Python test
->   case. Since Python2 is no longer officially supported by January 1, 2020, it
->   is recommended that subsequent test case development be guaranteed to run
->   correctly on Python3. For Python2, please consider being compatible if
->   appropriate without additional
->   burden. <https://nakedsecurity.sophos.com/2020/01/03/python-is-dead-long-live-python/> 
-
 ### CI submission adoption principle.
 
 -   Every commit / PR compilation must pass. Currently, the warning is treated

tests/pytest/account/account_create.py

@@ -19,9 +19,9 @@ from util.sql import tdSql
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         try:

tests/pytest/account/account_del.py

@@ -19,9 +19,9 @@ from util.sql import tdSql
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         print("==========step1")

tests/pytest/alter/alter_stable.py

@@ -9,9 +9,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
         self.types = [
             "int",
             "bigint",

tests/pytest/alter/alter_table.py

@@ -9,9 +9,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
         self.types = [
             "int",
             "bigint",

tests/pytest/alter/file_corrupt.py

@@ -19,9 +19,9 @@ from util.sql import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         tdSql.prepare()

tests/pytest/dbmgmt/database-name-boundary.py

@@ -23,9 +23,9 @@ from util.sql import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
 

tests/pytest/dbmgmt/dropDB_memory_test.py

@@ -19,9 +19,9 @@ from util.sql import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         tdSql.prepare()

tests/pytest/fulltest.sh

 #!/bin/bash
-python3 ./test.py $1 -f insert/basic.py
-python3 ./test.py $1 -f insert/int.py
-python3 ./test.py $1 -f insert/float.py
-python3 ./test.py $1 -f insert/bigint.py
-python3 ./test.py $1 -f insert/bool.py
-python3 ./test.py $1 -f insert/double.py
-python3 ./test.py $1 -f insert/smallint.py
-python3 ./test.py $1 -f insert/tinyint.py
-python3 ./test.py $1 -f insert/date.py
-python3 ./test.py $1 -f insert/binary.py
-python3 ./test.py $1 -f insert/nchar.py
-python3 ./test.py $1 -f insert/nchar-boundary.py
-python3 ./test.py $1 -f insert/nchar-unicode.py
-python3 ./test.py $1 -f insert/multi.py
+python3 ./test.py -f insert/basic.py
+python3 ./test.py -f insert/int.py
+python3 ./test.py -f insert/float.py
+python3 ./test.py -f insert/bigint.py
+python3 ./test.py -f insert/bool.py
+python3 ./test.py -f insert/double.py
+python3 ./test.py -f insert/smallint.py
+python3 ./test.py -f insert/tinyint.py
+python3 ./test.py -f insert/date.py
+python3 ./test.py -f insert/binary.py
+python3 ./test.py -f insert/nchar.py
+python3 ./test.py -f insert/nchar-boundary.py
+python3 ./test.py -f insert/nchar-unicode.py
+python3 ./test.py -f insert/multi.py
+python3 ./test.py -f insert/randomNullCommit.py
 
-python3 ./test.py $1 -f table/column_name.py
-python3 ./test.py $1 -f table/column_num.py
-python3 ./test.py $1 -f table/db_table.py
-python3 ./test.py $1 -f table/tablename-boundary.py
+python3 ./test.py -f table/column_name.py
+python3 ./test.py -f table/column_num.py
+python3 ./test.py -f table/db_table.py
+python3 ./test.py -f table/tablename-boundary.py
 
 # tag
-python3 ./test.py $1 -f tag_lite/filter.py
-python3 ./test.py $1 -f tag_lite/create-tags-boundary.py
-python3 ./test.py $1 -f tag_lite/3.py
-python3 ./test.py $1 -f tag_lite/4.py
-python3 ./test.py $1 -f tag_lite/5.py
-python3 ./test.py $1 -f tag_lite/6.py
-python3 ./test.py $1 -f tag_lite/add.py
-python3 ./test.py $1 -f tag_lite/bigint.py
-python3 ./test.py $1 -f tag_lite/binary_binary.py
-python3 ./test.py $1 -f tag_lite/binary.py
-python3 ./test.py $1 -f tag_lite/bool_binary.py
-python3 ./test.py $1 -f tag_lite/bool_int.py
-python3 ./test.py $1 -f tag_lite/bool.py
-python3 ./test.py $1 -f tag_lite/change.py
-python3 ./test.py $1 -f tag_lite/column.py
-python3 ./test.py $1 -f tag_lite/commit.py
-python3 ./test.py $1 -f tag_lite/create.py
-python3 ./test.py $1 -f tag_lite/datatype.py
-python3 ./test.py $1 -f tag_lite/datatype-without-alter.py
-python3 ./test.py $1 -f tag_lite/delete.py
-python3 ./test.py $1 -f tag_lite/double.py
-python3 ./test.py $1 -f tag_lite/float.py
-python3 ./test.py $1 -f tag_lite/int_binary.py
-python3 ./test.py $1 -f tag_lite/int_float.py
-python3 ./test.py $1 -f tag_lite/int.py
-python3 ./test.py $1 -f tag_lite/set.py
-python3 ./test.py $1 -f tag_lite/smallint.py
-python3 ./test.py $1 -f tag_lite/tinyint.py
+python3 ./test.py -f tag_lite/filter.py
+python3 ./test.py -f tag_lite/create-tags-boundary.py
+python3 ./test.py -f tag_lite/3.py
+python3 ./test.py -f tag_lite/4.py
+python3 ./test.py -f tag_lite/5.py
+python3 ./test.py -f tag_lite/6.py
+python3 ./test.py -f tag_lite/add.py
+python3 ./test.py -f tag_lite/bigint.py
+python3 ./test.py -f tag_lite/binary_binary.py
+python3 ./test.py -f tag_lite/binary.py
+python3 ./test.py -f tag_lite/bool_binary.py
+python3 ./test.py -f tag_lite/bool_int.py
+python3 ./test.py -f tag_lite/bool.py
+python3 ./test.py -f tag_lite/change.py
+python3 ./test.py -f tag_lite/column.py
+python3 ./test.py -f tag_lite/commit.py
+python3 ./test.py -f tag_lite/create.py
+python3 ./test.py -f tag_lite/datatype.py
+python3 ./test.py -f tag_lite/datatype-without-alter.py
+python3 ./test.py -f tag_lite/delete.py
+python3 ./test.py -f tag_lite/double.py
+python3 ./test.py -f tag_lite/float.py
+python3 ./test.py -f tag_lite/int_binary.py
+python3 ./test.py -f tag_lite/int_float.py
+python3 ./test.py -f tag_lite/int.py
+python3 ./test.py -f tag_lite/set.py
+python3 ./test.py -f tag_lite/smallint.py
+python3 ./test.py -f tag_lite/tinyint.py
 
-python3 ./test.py $1 -f dbmgmt/database-name-boundary.py
+python3 ./test.py -f dbmgmt/database-name-boundary.py
 
-python3 ./test.py $1 -f import_merge/importBlock1HO.py
-python3 ./test.py $1 -f import_merge/importBlock1HPO.py
-python3 ./test.py $1 -f import_merge/importBlock1H.py
-python3 ./test.py $1 -f import_merge/importBlock1S.py
-python3 ./test.py $1 -f import_merge/importBlock1Sub.py
-python3 ./test.py $1 -f import_merge/importBlock1TO.py
-python3 ./test.py $1 -f import_merge/importBlock1TPO.py
-python3 ./test.py $1 -f import_merge/importBlock1T.py
-python3 ./test.py $1 -f import_merge/importBlock2HO.py
-python3 ./test.py $1 -f import_merge/importBlock2HPO.py
-python3 ./test.py $1 -f import_merge/importBlock2H.py
-python3 ./test.py $1 -f import_merge/importBlock2S.py
-python3 ./test.py $1 -f import_merge/importBlock2Sub.py
-python3 ./test.py $1 -f import_merge/importBlock2TO.py
-python3 ./test.py $1 -f import_merge/importBlock2TPO.py
-python3 ./test.py $1 -f import_merge/importBlock2T.py
-python3 ./test.py $1 -f import_merge/importBlockbetween.py
-python3 ./test.py $1 -f import_merge/importCacheFileHO.py
-python3 ./test.py $1 -f import_merge/importCacheFileHPO.py
-python3 ./test.py $1 -f import_merge/importCacheFileH.py
-python3 ./test.py $1 -f import_merge/importCacheFileS.py
-python3 ./test.py $1 -f import_merge/importCacheFileSub.py
-python3 ./test.py $1 -f import_merge/importCacheFileTO.py
-python3 ./test.py $1 -f import_merge/importCacheFileTPO.py
-python3 ./test.py $1 -f import_merge/importCacheFileT.py
-python3 ./test.py $1 -f import_merge/importDataH2.py
-python3 ./test.py $1 -f import_merge/importDataHO2.py
-python3 ./test.py $1 -f import_merge/importDataHO.py
-python3 ./test.py $1 -f import_merge/importDataHPO.py
-python3 ./test.py $1 -f import_merge/importDataLastHO.py
-python3 ./test.py $1 -f import_merge/importDataLastHPO.py
-python3 ./test.py $1 -f import_merge/importDataLastH.py
-python3 ./test.py $1 -f import_merge/importDataLastS.py
-python3 ./test.py $1 -f import_merge/importDataLastSub.py
-python3 ./test.py $1 -f import_merge/importDataLastTO.py
-python3 ./test.py $1 -f import_merge/importDataLastTPO.py
-python3 ./test.py $1 -f import_merge/importDataLastT.py
-python3 ./test.py $1 -f import_merge/importDataS.py
-python3 ./test.py $1 -f import_merge/importDataSub.py
-python3 ./test.py $1 -f import_merge/importDataTO.py
-python3 ./test.py $1 -f import_merge/importDataTPO.py
-python3 ./test.py $1 -f import_merge/importDataT.py
-python3 ./test.py $1 -f import_merge/importHeadOverlap.py
-python3 ./test.py $1 -f import_merge/importHeadPartOverlap.py
-python3 ./test.py $1 -f import_merge/importHead.py
-python3 ./test.py $1 -f import_merge/importHORestart.py
-python3 ./test.py $1 -f import_merge/importHPORestart.py
-python3 ./test.py $1 -f import_merge/importHRestart.py
-python3 ./test.py $1 -f import_merge/importLastHO.py
-python3 ./test.py $1 -f import_merge/importLastHPO.py
-python3 ./test.py $1 -f import_merge/importLastH.py
-python3 ./test.py $1 -f import_merge/importLastS.py
-python3 ./test.py $1 -f import_merge/importLastSub.py
-python3 ./test.py $1 -f import_merge/importLastTO.py
-python3 ./test.py $1 -f import_merge/importLastTPO.py
-python3 ./test.py $1 -f import_merge/importLastT.py
-python3 ./test.py $1 -f import_merge/importSpan.py
-python3 ./test.py $1 -f import_merge/importSRestart.py
-python3 ./test.py $1 -f import_merge/importSubRestart.py
-python3 ./test.py $1 -f import_merge/importTailOverlap.py
-python3 ./test.py $1 -f import_merge/importTailPartOverlap.py
-python3 ./test.py $1 -f import_merge/importTail.py
-python3 ./test.py $1 -f import_merge/importToCommit.py
-python3 ./test.py $1 -f import_merge/importTORestart.py
-python3 ./test.py $1 -f import_merge/importTPORestart.py
-python3 ./test.py $1 -f import_merge/importTRestart.py
+python3 ./test.py -f import_merge/importBlock1HO.py
+python3 ./test.py -f import_merge/importBlock1HPO.py
+python3 ./test.py -f import_merge/importBlock1H.py
+python3 ./test.py -f import_merge/importBlock1S.py
+python3 ./test.py -f import_merge/importBlock1Sub.py
+python3 ./test.py -f import_merge/importBlock1TO.py
+python3 ./test.py -f import_merge/importBlock1TPO.py
+python3 ./test.py -f import_merge/importBlock1T.py
+python3 ./test.py -f import_merge/importBlock2HO.py
+python3 ./test.py -f import_merge/importBlock2HPO.py
+python3 ./test.py -f import_merge/importBlock2H.py
+python3 ./test.py -f import_merge/importBlock2S.py
+python3 ./test.py -f import_merge/importBlock2Sub.py
+python3 ./test.py -f import_merge/importBlock2TO.py
+python3 ./test.py -f import_merge/importBlock2TPO.py
+python3 ./test.py -f import_merge/importBlock2T.py
+python3 ./test.py -f import_merge/importBlockbetween.py
+python3 ./test.py -f import_merge/importCacheFileHO.py
+python3 ./test.py -f import_merge/importCacheFileHPO.py
+python3 ./test.py -f import_merge/importCacheFileH.py
+python3 ./test.py -f import_merge/importCacheFileS.py
+python3 ./test.py -f import_merge/importCacheFileSub.py
+python3 ./test.py -f import_merge/importCacheFileTO.py
+python3 ./test.py -f import_merge/importCacheFileTPO.py
+python3 ./test.py -f import_merge/importCacheFileT.py
+python3 ./test.py -f import_merge/importDataH2.py
+python3 ./test.py -f import_merge/importDataHO2.py
+python3 ./test.py -f import_merge/importDataHO.py
+python3 ./test.py -f import_merge/importDataHPO.py
+python3 ./test.py -f import_merge/importDataLastHO.py
+python3 ./test.py -f import_merge/importDataLastHPO.py
+python3 ./test.py -f import_merge/importDataLastH.py
+python3 ./test.py -f import_merge/importDataLastS.py
+python3 ./test.py -f import_merge/importDataLastSub.py
+python3 ./test.py -f import_merge/importDataLastTO.py
+python3 ./test.py -f import_merge/importDataLastTPO.py
+python3 ./test.py -f import_merge/importDataLastT.py
+python3 ./test.py -f import_merge/importDataS.py
+python3 ./test.py -f import_merge/importDataSub.py
+python3 ./test.py -f import_merge/importDataTO.py
+python3 ./test.py -f import_merge/importDataTPO.py
+python3 ./test.py -f import_merge/importDataT.py
+python3 ./test.py -f import_merge/importHeadOverlap.py
+python3 ./test.py -f import_merge/importHeadPartOverlap.py
+python3 ./test.py -f import_merge/importHead.py
+python3 ./test.py -f import_merge/importHORestart.py
+python3 ./test.py -f import_merge/importHPORestart.py
+python3 ./test.py -f import_merge/importHRestart.py
+python3 ./test.py -f import_merge/importLastHO.py
+python3 ./test.py -f import_merge/importLastHPO.py
+python3 ./test.py -f import_merge/importLastH.py
+python3 ./test.py -f import_merge/importLastS.py
+python3 ./test.py -f import_merge/importLastSub.py
+python3 ./test.py -f import_merge/importLastTO.py
+python3 ./test.py -f import_merge/importLastTPO.py
+python3 ./test.py -f import_merge/importLastT.py
+python3 ./test.py -f import_merge/importSpan.py
+python3 ./test.py -f import_merge/importSRestart.py
+python3 ./test.py -f import_merge/importSubRestart.py
+python3 ./test.py -f import_merge/importTailOverlap.py
+python3 ./test.py -f import_merge/importTailPartOverlap.py
+python3 ./test.py -f import_merge/importTail.py
+python3 ./test.py -f import_merge/importToCommit.py
+python3 ./test.py -f import_merge/importTORestart.py
+python3 ./test.py -f import_merge/importTPORestart.py
+python3 ./test.py -f import_merge/importTRestart.py
+python3 ./test.py -f import_merge/importInsertThenImport.py
 
 # user
-python3 ./test.py $1 -f user/user_create.py
-python3 ./test.py $1 -f user/pass_len.py
+python3 ./test.py -f user/user_create.py
+python3 ./test.py -f user/pass_len.py
 
 # table
-#python3 ./test.py $1 -f table/del_stable.py
+#python3 ./test.py -f table/del_stable.py
+
+#query
+python3 ./test.py -f query/filter.py
 

tests/pytest/import_merge/importBlock1H.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock1HO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock1HPO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock1S.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock1Sub.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock1T.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock1TO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock1TPO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock2H.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock2HO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock2HPO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock2S.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock2Sub.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock2T.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock2TO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock2TPO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlockbetween.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importCacheFileH.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importCacheFileHO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importCacheFileHPO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importCacheFileS.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importCacheFileSub.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importCacheFileT.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importCacheFileTO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importCacheFileTPO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importDataH2.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importDataHO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importDataHO2.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1