Merge remote-tracking branch 'origin/feature/query' into feature/query

src/client/inc/tscProfile.h

@@ -26,7 +26,7 @@ void tscAddIntoSqlList(SSqlObj *pSql);
 void tscRemoveFromSqlList(SSqlObj *pSql);
 void tscAddIntoStreamList(SSqlStream *pStream);
 void tscRemoveFromStreamList(SSqlStream *pStream, SSqlObj *pSqlObj);
-char *tscBuildQueryStreamDesc(char *pMsg, STscObj *pObj);
+int  tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj);
 void tscKillQuery(STscObj *pObj, uint32_t killId);
 void tscKillStream(STscObj *pObj, uint32_t killId);
 void tscKillConnection(STscObj *pObj);

src/client/inc/tsclient.h

@@ -294,6 +294,7 @@ typedef struct STscObj {
   char               sversion[TSDB_VERSION_LEN];
   char               writeAuth : 1;
   char               superAuth : 1;
+  uint32_t           connId;
   struct SSqlObj *   pHb;
   struct SSqlObj *   sqlList;
   struct SSqlStream *streamList;

src/client/src/tscProfile.c

@@ -19,6 +19,7 @@
 #include "ttime.h"
 #include "ttimer.h"
 #include "tutil.h"
+#include "taosmsg.h"
 
 void  tscSaveSlowQueryFp(void *handle, void *tmrId);
 void *tscSlowQueryConn = NULL;
@@ -96,7 +97,7 @@ void tscSaveSlowQuery(SSqlObj *pSql) {
   }
 
   tscTrace("%p query time:%" PRId64 " sql:%s", pSql, pSql->res.useconds, pSql->sqlstr);
-  int32_t sqlSize = TSDB_SHOW_SQL_LEN + size;
+  int32_t sqlSize = TSDB_SLOW_QUERY_SQL_LEN + size;
   
   char *sql = malloc(sqlSize);
   if (sql == NULL) {
@@ -106,9 +107,9 @@ void tscSaveSlowQuery(SSqlObj *pSql) {
   
   int len = snprintf(sql, size, "insert into %s.slowquery values(now, '%s', %" PRId64 ", %" PRId64 ", '", tsMonitorDbName,
           pSql->pTscObj->user, pSql->stime, pSql->res.useconds);
-  int sqlLen = snprintf(sql + len, TSDB_SHOW_SQL_LEN, "%s", pSql->sqlstr);
-  if (sqlLen > TSDB_SHOW_SQL_LEN - 1) {
-    sqlLen = len + TSDB_SHOW_SQL_LEN - 1;
+  int sqlLen = snprintf(sql + len, TSDB_SLOW_QUERY_SQL_LEN, "%s", pSql->sqlstr);
+  if (sqlLen > TSDB_SLOW_QUERY_SQL_LEN - 1) {
+    sqlLen = len + TSDB_SLOW_QUERY_SQL_LEN - 1;
   } else {
     sqlLen += len;
   }
@@ -205,28 +206,28 @@ void tscKillStream(STscObj *pObj, uint32_t killId) {
   }
 
   pthread_mutex_unlock(&pObj->mutex);
-  
+
   if (pStream) {
     tscTrace("%p stream:%p is killed, streamId:%d", pStream->pSql, pStream, killId);
+    if (pStream->callback) {
+      pStream->callback(pStream->param);
+    }
+    taos_close_stream(pStream);
+  } else {
+    tscError("failed to kill stream, streamId:%d not exist", killId);
   }
-
-  if (pStream->callback) {
-    pStream->callback(pStream->param);
-  }
-  taos_close_stream(pStream);
 }
 
-char *tscBuildQueryStreamDesc(char *pMsg, STscObj *pObj) {
-  char *  pMax = pMsg + TSDB_PAYLOAD_SIZE - 256;
+int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
+  SCMHeartBeatMsg *pHeartbeat = pMsg;
+  int allocedQueriesNum = pHeartbeat->numOfQueries;
+  int allocedStreamsNum = pHeartbeat->numOfStreams;
   
-  SQqueryList *pQList = (SQqueryList *)pMsg;
-  pQList->numOfQueries = 0;
-
-  SQueryDesc *pQdesc = (SQueryDesc*)(pMsg + sizeof(SQqueryList));
+  pHeartbeat->numOfQueries = 0;
+  SQueryDesc *pQdesc = (SQueryDesc *)pHeartbeat->pData;
 
   // We extract the lock to tscBuildHeartBeatMsg function.
-  /* pthread_mutex_lock (&pObj->mutex); */
-  pMsg += sizeof(SQqueryList);
+
   SSqlObj *pSql = pObj->sqlList;
   while (pSql) {
     /*
@@ -240,47 +241,46 @@ char *tscBuildQueryStreamDesc(char *pMsg, STscObj *pObj) {
 
     strncpy(pQdesc->sql, pSql->sqlstr, TSDB_SHOW_SQL_LEN - 1);
     pQdesc->sql[TSDB_SHOW_SQL_LEN - 1] = 0;
-    pQdesc->stime = pSql->stime;
-    pQdesc->queryId = pSql->queryId;
-    pQdesc->useconds = pSql->res.useconds;
+    pQdesc->stime = htobe64(pSql->stime);
+    pQdesc->queryId = htonl(pSql->queryId);
+    pQdesc->useconds = htobe64(pSql->res.useconds);
 
-    pQList->numOfQueries++;
+    pHeartbeat->numOfQueries++;
     pQdesc++;
     pSql = pSql->next;
-    pMsg += sizeof(SQueryDesc);
-    if (pMsg > pMax) break;
+    if (pHeartbeat->numOfQueries >= allocedQueriesNum) break;
   }
 
-  SStreamList *pSList = (SStreamList *)pMsg;
-  pSList->numOfStreams = 0;
-  
-  SStreamDesc *pSdesc = (SStreamDesc*) (pMsg + sizeof(SStreamList));
+  pHeartbeat->numOfStreams = 0;
+  SStreamDesc *pSdesc = (SStreamDesc *)pQdesc;
 
-  pMsg += sizeof(SStreamList);
   SSqlStream *pStream = pObj->streamList;
   while (pStream) {
     strncpy(pSdesc->sql, pStream->pSql->sqlstr, TSDB_SHOW_SQL_LEN - 1);
     pSdesc->sql[TSDB_SHOW_SQL_LEN - 1] = 0;
-    pSdesc->streamId = pStream->streamId;
-    pSdesc->num = pStream->num;
+    pSdesc->streamId = htonl(pStream->streamId);
+    pSdesc->num = htobe64(pStream->num);
 
-    pSdesc->useconds = pStream->useconds;
-    pSdesc->stime = pStream->stime - pStream->interval;
-    pSdesc->ctime = pStream->ctime;
+    pSdesc->useconds = htobe64(pStream->useconds);
+    pSdesc->stime = htobe64(pStream->stime - pStream->interval);
+    pSdesc->ctime = htobe64(pStream->ctime);
 
-    pSdesc->slidingTime = pStream->slidingTime;
-    pSdesc->interval = pStream->interval;
+    pSdesc->slidingTime = htobe64(pStream->slidingTime);
+    pSdesc->interval = htobe64(pStream->interval);
 
-    pSList->numOfStreams++;
+    pHeartbeat->numOfStreams++;
     pSdesc++;
     pStream = pStream->next;
-    pMsg += sizeof(SStreamDesc);
-    if (pMsg > pMax) break;
+    if (pHeartbeat->numOfStreams >= allocedStreamsNum) break;
   }
 
-  /* pthread_mutex_unlock (&pObj->mutex); */
+  int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) +
+                   sizeof(SCMHeartBeatMsg);
+  pHeartbeat->connId = htonl(pObj->connId);
+  pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries);
+  pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams);
 
-  return pMsg;
+  return msgLen;
 }
 
 void tscKillConnection(STscObj *pObj) {

src/client/src/tscSQLParser.c

@@ -93,7 +93,7 @@ static int32_t validateArithmeticSQLExpr(tSQLExpr* pExpr, SQueryInfo* pQueryInfo
 static int32_t validateDNodeConfig(tDCLSQL* pOptions);
 static int32_t validateLocalConfig(tDCLSQL* pOptions);
 static int32_t validateColumnName(char* name);
-static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo);
+static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType);
 
 static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField);
 static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo);
@@ -531,7 +531,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     case TSDB_SQL_KILL_QUERY:
     case TSDB_SQL_KILL_STREAM:
     case TSDB_SQL_KILL_CONNECTION: {
-      if ((code = setKillInfo(pSql, pInfo)) != TSDB_CODE_SUCCESS) {
+      if ((code = setKillInfo(pSql, pInfo, pInfo->type)) != TSDB_CODE_SUCCESS) {
         return code;
       }
 
@@ -2229,37 +2229,45 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
-  const char* msg1 = "invalid ip address";
-  const char* msg2 = "invalid port";
+int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) {
+  const char* msg1 = "invalid connection ID";
+  const char* msg2 = "invalid query ID";
+  const char* msg3 = "invalid stream ID";
 
   SSqlCmd* pCmd = &pSql->cmd;
   pCmd->command = pInfo->type;
-
-  SSQLToken* ip = &(pInfo->pDCLInfo->ip);
-  if (ip->n > TSDB_KILL_MSG_LEN) {
+  
+  SSQLToken* idStr = &(pInfo->pDCLInfo->ip);
+  if (idStr->n > TSDB_KILL_MSG_LEN) {
     return TSDB_CODE_INVALID_SQL;
   }
 
-  strncpy(pCmd->payload, ip->z, ip->n);
+  strncpy(pCmd->payload, idStr->z, idStr->n);
 
   const char delim = ':';
+  char* connIdStr = strtok(idStr->z, &delim);
+  char* queryIdStr = strtok(NULL, &delim);
 
-  char* ipStr = strtok(ip->z, &delim);
-  char* portStr = strtok(NULL, &delim);
-
-  if (!validateIpAddress(ipStr, strlen(ipStr))) {
+  int32_t connId = (int32_t)strtol(connIdStr, NULL, 10);
+  if (connId <= 0) {
     memset(pCmd->payload, 0, strlen(pCmd->payload));
-
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
   }
 
-  uint16_t port = (uint16_t)strtol(portStr, NULL, 10);
-  if (port <= 0 || port > 65535) {
-    memset(pCmd->payload, 0, strlen(pCmd->payload));
-    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
+  if (killType == TSDB_SQL_KILL_CONNECTION) {
+    return TSDB_CODE_SUCCESS;
   }
 
+  int32_t queryId = (int32_t)strtol(queryIdStr, NULL, 10);
+  if (queryId <= 0) {
+    memset(pCmd->payload, 0, strlen(pCmd->payload));
+    if (killType == TSDB_SQL_KILL_QUERY) {
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
+    } else {
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
+    }
+  }
+  
   return TSDB_CODE_SUCCESS;
 }
 

src/client/src/tscServer.c

@@ -114,6 +114,8 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
     if (pIpList->numOfIps > 0) 
       tscSetMgmtIpList(pIpList);
 
+    pSql->pTscObj->connId = htonl(pRsp->connId);
+
     if (pRsp->killConnection) {
       tscKillConnection(pObj);
     } else {
@@ -1129,13 +1131,6 @@ int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SSqlCmd *pCmd = &pSql->cmd;
   pCmd->payloadLen = sizeof(SCMKillQueryMsg);
 
-  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
-    tscError("%p failed to malloc for query msg", pSql);
-    return TSDB_CODE_CLI_OUT_OF_MEMORY;
-  }
-
-  SCMKillQueryMsg *pKill = (SCMKillQueryMsg*)pCmd->payload;
-  strncpy(pKill->queryId, pInfo->pDCLInfo->ip.z, pInfo->pDCLInfo->ip.n);
   switch (pCmd->command) {
     case TSDB_SQL_KILL_QUERY:
       pCmd->msgType = TSDB_MSG_TYPE_CM_KILL_QUERY;
@@ -1743,57 +1738,43 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
-int tscEstimateHeartBeatMsgLength(SSqlObj *pSql) {
-  int      size = 0;
+int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
+  SSqlCmd *pCmd = &pSql->cmd;
   STscObj *pObj = pSql->pTscObj;
 
-  size += tsRpcHeadSize;
-  size += sizeof(SQqueryList);
+  pthread_mutex_lock(&pObj->mutex);
 
+  int32_t numOfQueries = 2;
   SSqlObj *tpSql = pObj->sqlList;
   while (tpSql) {
-    size += sizeof(SQueryDesc);
     tpSql = tpSql->next;
+    numOfQueries++;
   }
 
-  size += sizeof(SStreamList);
+  int32_t numOfStreams = 2;
   SSqlStream *pStream = pObj->streamList;
   while (pStream) {
-    size += sizeof(SStreamDesc);
     pStream = pStream->next;
+    numOfStreams++;
   }
 
-  return size + TSDB_EXTRA_PAYLOAD_SIZE;
-}
-
-int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
-  char *pMsg, *pStart;
-  int   msgLen = 0;
-  int   size = 0;
-
-  SSqlCmd *pCmd = &pSql->cmd;
-  STscObj *pObj = pSql->pTscObj;
-
-  pthread_mutex_lock(&pObj->mutex);
-
-  size = tscEstimateHeartBeatMsgLength(pSql);
+  int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SCMHeartBeatMsg) + 100;
   if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) {
     pthread_mutex_unlock(&pObj->mutex);
     tscError("%p failed to malloc for heartbeat msg", pSql);
     return -1;
   }
 
-  pMsg = pCmd->payload;
-  pStart = pMsg;
+  SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload;
+  pHeartbeat->numOfQueries = numOfQueries;
+  pHeartbeat->numOfStreams = numOfStreams;
+  int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj);
 
-  pMsg = tscBuildQueryStreamDesc(pMsg, pObj);
   pthread_mutex_unlock(&pObj->mutex);
 
-  msgLen = pMsg - pStart;
   pCmd->payloadLen = msgLen;
   pCmd->msgType = TSDB_MSG_TYPE_CM_HEARTBEAT;
 
-  assert(msgLen + minMsgSize() <= size);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -2204,6 +2185,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
   strcpy(pObj->sversion, pConnect->serverVersion);
   pObj->writeAuth = pConnect->writeAuth;
   pObj->superAuth = pConnect->superAuth;
+  pObj->connId = htonl(pConnect->connId);
   taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);
 
   return 0;

src/common/inc/tdataformat.h

@@ -251,7 +251,7 @@ void       tdFreeDataCols(SDataCols *pCols);
 void       tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
 void       tdPopDataColsPoints(SDataCols *pCols, int pointsToPop);  //!!!!
 int        tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
-void       tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows);
+void       tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows);
 
 // ----------------- K-V data row structure
 /*

src/common/src/tdataformat.c

@@ -450,7 +450,8 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
 
     int iter1 = 0;
     int iter2 = 0;
-    tdMergeTwoDataCols(target, pTarget, &iter1, source, &iter2, pTarget->numOfRows + rowsToMerge);
+    tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, &iter2, source->numOfRows,
+                       pTarget->numOfRows + rowsToMerge);
   }
 
   tdFreeDataCols(pTarget);
@@ -461,15 +462,15 @@ _err:
   return -1;
 }
 
-void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows) {
-  // TODO: add resolve duplicate key here
+void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) {
   tdResetDataCols(target);
+  ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
 
   while (target->numOfRows < tRows) {
-    if (*iter1 >= src1->numOfRows && *iter2 >= src2->numOfRows) break;
+    if (*iter1 >= limit1 && *iter2 >= limit2) break;
 
-    TSKEY key1 = (*iter1 >= src1->numOfRows) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
-    TSKEY key2 = (*iter2 >= src2->numOfRows) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
+    TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
+    TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
 
     if (key1 <= key2) {
       for (int i = 0; i < src1->numOfCols; i++) {

src/dnode/src/dnodeMain.c

@@ -46,7 +46,7 @@ typedef struct {
   void              (*cleanup)();
 } SDnodeComponent;
 
-static const SDnodeComponent SDnodeComponents[] = {
+static const SDnodeComponent tsDnodeComponents[] = {
   {"storage", dnodeInitStorage,    dnodeCleanupStorage},
   {"vread",   dnodeInitVnodeRead,  dnodeCleanupVnodeRead},
   {"vwrite",  dnodeInitVnodeWrite, dnodeCleanupVnodeWrite},
@@ -72,14 +72,14 @@ static int dnodeCreateDir(const char *dir) {
 
 static void dnodeCleanupComponents(int32_t stepId) {
   for (int32_t i = stepId; i >= 0; i--) {
-    SDnodeComponents[i].cleanup();
+    tsDnodeComponents[i].cleanup();
   }
 }
 
 static int32_t dnodeInitComponents() {
   int32_t code = 0;
-  for (int32_t i = 0; i < sizeof(SDnodeComponents) / sizeof(SDnodeComponents[0]); i++) {
-    if (SDnodeComponents[i].init() != 0) {
+  for (int32_t i = 0; i < sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]); i++) {
+    if (tsDnodeComponents[i].init() != 0) {
       dnodeCleanupComponents(i);
       code = -1;
       break;
@@ -133,7 +133,7 @@ int32_t dnodeInitSystem() {
 void dnodeCleanUpSystem() {
   if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) {
     dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
-    dnodeCleanupComponents(sizeof(SDnodeComponents) / sizeof(SDnodeComponents[0]) - 1);
+    dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1);
     taos_cleanup();
     taosCloseLog();
   }

src/inc/taosdef.h

@@ -231,7 +231,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_SHELL_VNODE_BITS     24
 #define TSDB_SHELL_SID_MASK       0xFF
 #define TSDB_HTTP_TOKEN_LEN       20
-#define TSDB_SHOW_SQL_LEN         512
+#define TSDB_SHOW_SQL_LEN         64
+#define TSDB_SLOW_QUERY_SQL_LEN   512
 
 #define TSDB_METER_STATE_OFFLINE  0
 #define TSDB_METER_STATE_ONLLINE  1

src/inc/taoserror.h

@@ -150,6 +150,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS,        0, 0x0405, "no disk perm
 TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED,             0, 0x0406, "file corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED,           0, 0x0407, "memory corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUCH_FILE_OR_DIR,       0, 0x0408, "no such file or directory")
+TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_SHELL_CONNS,       0, 0x0409, "too many shell conns")
 
 // client
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION,     0, 0x0481, "invalid client version")

src/inc/taosmsg.h

@@ -137,6 +137,7 @@ enum _mgmt_table {
   TSDB_MGMT_TABLE_SCORES,
   TSDB_MGMT_TABLE_GRANTS,
   TSDB_MGMT_TABLE_VNODES,
+  TSDB_MGMT_TABLE_STREAMTABLES,
   TSDB_MGMT_TABLE_MAX,
 };
 
@@ -299,6 +300,9 @@ typedef struct {
   char      serverVersion[TSDB_VERSION_LEN];
   int8_t    writeAuth;
   int8_t    superAuth;
+  int8_t    reserved1;
+  int8_t    reserved2;
+  int32_t   connId;
   SRpcIpSet ipList;
 } SCMConnectRsp;
 
@@ -716,16 +720,10 @@ typedef struct {
 } SStreamDesc;
 
 typedef struct {
-  int32_t    numOfQueries;
-} SQqueryList;
-
-typedef struct {
-  int32_t     numOfStreams;
-} SStreamList;
-
-typedef struct {
-  SQqueryList qlist;
-  SStreamList slist;
+  uint32_t connId;
+  int32_t  numOfQueries;
+  int32_t  numOfStreams;
+  char     pData[];
 } SCMHeartBeatMsg;
 
 typedef struct {
@@ -733,6 +731,7 @@ typedef struct {
   uint32_t  streamId;
   uint32_t  totalDnodes;
   uint32_t  onlineDnodes;
+  uint32_t  connId;
   int8_t    killConnection;
   SRpcIpSet ipList;
 } SCMHeartBeatRsp;

src/inc/ttokendef.h

@@ -148,8 +148,8 @@
 #define TK_SET                            130
 #define TK_KILL                           131
 #define TK_CONNECTION                     132
-#define TK_COLON                          133
-#define TK_STREAM                         134
+#define TK_STREAM                         133
+#define TK_COLON                          134
 #define TK_ABORT                          135
 #define TK_AFTER                          136
 #define TK_ATTACH                         137

src/mnode/inc/mnodeDef.h

@@ -182,8 +182,6 @@ typedef struct SUserObj {
   int8_t            updateEnd[1];
   int32_t           refCount;
   struct SAcctObj * pAcct;
-  SQqueryList *     pQList;  // query list
-  SStreamList *     pSList;  // stream list
 } SUserObj;
 
 typedef struct {

src/mnode/inc/mnodeProfile.h

@@ -21,9 +21,30 @@ extern "C" {
 #endif
 #include "mnodeDef.h"
 
+typedef struct {
+  char     user[TSDB_USER_LEN + 1];
+  int8_t   killed;
+  uint16_t port;
+  uint32_t ip;
+  uint32_t connId;
+  uint64_t stime;
+  uint64_t lastAccess;
+  uint32_t queryId;
+  uint32_t streamId;
+  int32_t  numOfQueries;
+  int32_t  numOfStreams;
+  SStreamDesc *pStreams;
+  SQueryDesc * pQueries;
+} SConnObj;
+
 int32_t mnodeInitProfile();
 void    mnodeCleanupProfile();
 
+SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port);
+SConnObj *mnodeAccquireConn(uint32_t connId, char *user, uint32_t ip, uint16_t port);
+void      mnodeReleaseConn(SConnObj *pConn);
+int32_t   mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg);
+
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mnodeShow.h

@@ -28,6 +28,7 @@ typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pCon
 typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 void mnodeAddShowMetaHandle(uint8_t showType, SShowMetaFp fp);
 void mnodeAddShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp);
+void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow);
 
 #ifdef __cplusplus
 }

src/mnode/src/mnodeMain.c

@@ -33,14 +33,54 @@
 #include "mnodeUser.h"
 #include "mnodeTable.h"
 #include "mnodeShow.h"
+#include "mnodeProfile.h"
+
+typedef struct {
+  const char *const name;
+  int               (*init)();
+  void              (*cleanup)();
+} SMnodeComponent;
 
 void *tsMnodeTmr;
 static bool tsMgmtIsRunning = false;
 
+static const SMnodeComponent tsMnodeComponents[] = {
+  {"profile", mnodeInitProfile, mnodeCleanupProfile},
+  {"accts",   mnodeInitAccts,   mnodeCleanupAccts},
+  {"users",   mnodeInitUsers,   mnodeCleanupUsers},
+  {"dnodes",  mnodeInitDnodes,  mnodeCleanupDnodes},
+  {"dbs",     mnodeInitDbs,     mnodeCleanupDbs},
+  {"vgroups", mnodeInitVgroups, mnodeCleanupVgroups},
+  {"tables",  mnodeInitTables,  mnodeCleanupTables},  
+  {"mnodes",  mnodeInitMnodes,  mnodeCleanupMnodes},
+  {"sdb",     sdbInit,          sdbCleanUp},
+  {"balance", balanceInit,      balanceCleanUp},
+  {"grant",   grantInit,        grantCleanUp},
+  {"show",    mnodeInitShow,    mnodeCleanUpShow}
+};
+
 static void mnodeInitTimer();
 static void mnodeCleanupTimer();
 static bool mnodeNeedStart() ;
 
+static void mnodeCleanupComponents(int32_t stepId) {
+  for (int32_t i = stepId; i >= 0; i--) {
+    tsMnodeComponents[i].cleanup();
+  }
+}
+
+static int32_t mnodeInitComponents() {
+  int32_t code = 0;
+  for (int32_t i = 0; i < sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]); i++) {
+    if (tsMnodeComponents[i].init() != 0) {
+      mnodeCleanupComponents(i);
+      code = -1;
+      break;
+    }
+  }
+  return code;
+}
+
 int32_t mnodeStartSystem() {
   if (tsMgmtIsRunning) {
     mPrint("mnode module already started...");
@@ -57,57 +97,7 @@ int32_t mnodeStartSystem() {
   dnodeAllocateMnodeRqueue();
   dnodeAllocateMnodePqueue();
 
-  if (mnodeInitAccts() < 0) {
-    mError("failed to init accts");
-    return -1;
-  }
-
-  if (mnodeInitUsers() < 0) {
-    mError("failed to init users");
-    return -1;
-  }
-
-  if (mnodeInitDnodes() < 0) {
-    mError("failed to init dnodes");
-    return -1;
-  }
-
-  if (mnodeInitDbs() < 0) {
-    mError("failed to init dbs");
-    return -1;
-  }
-
-  if (mnodeInitVgroups() < 0) {
-    mError("failed to init vgroups");
-    return -1;
-  }
-
-  if (mnodeInitTables() < 0) {
-    mError("failed to init tables");
-    return -1;
-  }
-
-  if (mnodeInitMnodes() < 0) {
-    mError("failed to init mnodes");
-    return -1;
-  }
-
-  if (sdbInit() < 0) {
-    mError("failed to init sdb");
-    return -1;
-  }
-
-  if (balanceInit() < 0) {
-    mError("failed to init balance")
-  }
-
-  if (grantInit() < 0) {
-    mError("failed to init grant");
-    return -1;
-  }
-
-  if (mnodeInitShow() < 0) {
-    mError("failed to init show");
+  if (mnodeInitComponents() != 0) {
     return -1;
   }
 
@@ -115,7 +105,6 @@ int32_t mnodeStartSystem() {
   tsMgmtIsRunning = true;
 
   mPrint("mnode is initialized successfully");
-
   return 0;
 }
 
@@ -133,17 +122,8 @@ void mnodeCleanupSystem() {
   dnodeFreeMnodeRqueue();
   dnodeFreeMnodePqueue();
   mnodeCleanupTimer();
-  mnodeCleanUpShow();
-  grantCleanUp();
-  balanceCleanUp();
-  sdbCleanUp();
-  mnodeCleanupMnodes();
-  mnodeCleanupTables();
-  mnodeCleanupVgroups();
-  mnodeCleanupDbs();
-  mnodeCleanupDnodes();
-  mnodeCleanupUsers();
-  mnodeCleanupAccts();
+  mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1);
+
   mPrint("mnode is cleaned up");
 }
 

src/mnode/src/mnodeShow.c

@@ -227,21 +227,47 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
     return TSDB_CODE_SERV_OUT_OF_MEMORY;
   }
 
+  SCMHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont;
+  SRpcConnInfo connInfo;
+  rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo);
+    
+  int32_t connId = htonl(pHBMsg->connId);
+  SConnObj *pConn = mnodeAccquireConn(connId, connInfo.user, connInfo.clientIp, connInfo.clientPort);
+  if (pConn == NULL) {
+    pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort);
+  }
+
+  if (pConn == NULL) {
+    // do not close existing links, otherwise
+    // mError("failed to create connId, close connect");
+    // pHBRsp->killConnection = 1;
+  } else {
+    pHBRsp->connId = htonl(pConn->connId);
+    mnodeSaveQueryStreamList(pConn, pHBMsg);
+    
+    if (pConn->killed != 0) {
+      pHBRsp->killConnection = 1;
+    }
+
+    if (pConn->streamId != 0) {
+      pHBRsp->streamId = htonl(pConn->streamId);
+      pConn->streamId = 0;
+    }
+
+    if (pConn->queryId != 0) {
+      pHBRsp->queryId = htonl(pConn->queryId);
+      pConn->queryId = 0;
+    }
+  }
+
   pHBRsp->onlineDnodes = htonl(mnodeGetOnlinDnodesNum());
   pHBRsp->totalDnodes = htonl(mnodeGetDnodesNum());
   mnodeGetMnodeIpSetForShell(&pHBRsp->ipList);
-  
-  /*
-   * TODO
-   * Dispose kill stream or kill query message
-   */
-  pHBRsp->queryId = 0;
-  pHBRsp->streamId = 0;
-  pHBRsp->killConnection = 0;
 
   pMsg->rpcRsp.rsp = pHBRsp;
   pMsg->rpcRsp.len = sizeof(SCMHeartBeatRsp);
   
+  mnodeReleaseConn(pConn);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -281,11 +307,19 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
     goto connect_over;
   }
 
+  SConnObj *pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort);
+  if (pConn == NULL) {
+    code = terrno;
+  } else {
+    pConnectRsp->connId = htonl(pConn->connId);
+    mnodeReleaseConn(pConn);
+  }
+
   sprintf(pConnectRsp->acctId, "%x", pAcct->acctId);
   strcpy(pConnectRsp->serverVersion, version);
   pConnectRsp->writeAuth = pUser->writeAuth;
   pConnectRsp->superAuth = pUser->superAuth;
-
+  
   mnodeGetMnodeIpSetForShell(&pConnectRsp->ipList);
 
 connect_over:
@@ -358,3 +392,11 @@ static void mnodeReleaseShowObj(void *pShow, bool forceRemove) {
   mTrace("%p, show is released, force:%s", pShow, forceRemove ? "true" : "false");
   taosCacheRelease(tsMnodeShowCache, &pShow, forceRemove);
 }
+
+void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) {
+  if (rows < capacity) {
+    for (int32_t i = 0; i < numOfCols; ++i) {
+      memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows);
+    }
+  }
+}
\ No newline at end of file

src/mnode/src/mnodeTable.c

@@ -61,8 +61,8 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
 static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t rows, void *pConn);
  
 static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *mnodeMsg);
 static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg);
@@ -568,8 +568,8 @@ int32_t mnodeInitTables() {
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mnodeRetrieveShowTables);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mnodeGetShowSuperTableMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mnodeRetrieveShowSuperTables);
-  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMS, mnodeGetStreamMeta);
-  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMS, mnodeRetrieveStreams);
+  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeGetStreamTableMeta);
+  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeRetrieveStreamTables);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -2111,14 +2111,6 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   return 0;
 }
 
-static void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) {
-  if (rows < capacity) {
-    for (int32_t i = 0; i < numOfCols; ++i) {
-      memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows);
-    }
-  }
-}
-
 static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
   SDbObj *pDb = mnodeGetDb(pShow->db);
   if (pDb == NULL) return 0;
@@ -2262,7 +2254,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
   return code;
 }
 
-static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   SDbObj *pDb = mnodeGetDb(pShow->db);
   if (pDb == NULL) return TSDB_CODE_DB_NOT_SELECTED;
 
@@ -2308,7 +2300,7 @@ static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
   return 0;
 }
 
-static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
   SDbObj *pDb = mnodeGetDb(pShow->db);
   if (pDb == NULL) return 0;
 

src/plugins/monitor/src/monitorMain.c

@@ -183,7 +183,7 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
     snprintf(sql, SQL_LENGTH,
              "create table if not exists %s.slowquery(ts timestamp, username "
              "binary(%d), created_time timestamp, time bigint, sql binary(%d))",
-             tsMonitorDbName, TSDB_TABLE_ID_LEN, TSDB_SHOW_SQL_LEN);
+             tsMonitorDbName, TSDB_TABLE_ID_LEN, TSDB_SLOW_QUERY_SQL_LEN);
   } else if (cmd == MONITOR_CMD_CREATE_TB_LOG) {
     snprintf(sql, SQL_LENGTH,
              "create table if not exists %s.log(ts timestamp, level tinyint, "

src/tsdb/src/tsdbRWHelper.c

@@ -977,7 +977,8 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
       // tdResetDataCols(pHelper->pDataCols[1]);
       while (true) {
         if (iter1 >= pHelper->pDataCols[0]->numOfRows && iter2 >= rows3) break;
-        tdMergeTwoDataCols(pHelper->pDataCols[1], pHelper->pDataCols[0], &iter1, pDataCols, &iter2, pHelper->config.maxRowsPerFileBlock * 4 / 5);
+        tdMergeTwoDataCols(pHelper->pDataCols[1], pHelper->pDataCols[0], &iter1, pHelper->pDataCols[0]->numOfRows,
+                           pDataCols, &iter2, rowsWritten, pHelper->config.maxRowsPerFileBlock * 4 / 5);
         ASSERT(pHelper->pDataCols[1]->numOfRows > 0);
         if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pHelper->pDataCols[1],
                                  pHelper->pDataCols[1]->numOfRows, &compBlock, false, true) < 0)
@@ -989,54 +990,6 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
         }
         round++;
         blkIdx++;
-        // TODO: the blkIdx here is not correct
-
-        // if (iter1 >= pHelper->pDataCols[0]->numOfRows && iter2 >= rows3) {
-        //   if (pHelper->pDataCols[1]->numOfRows > 0) {
-        //     if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1],
-        //                              pHelper->pDataCols[1]->numOfRows, &compBlock, false, true) < 0)
-        //       goto _err;
-        //     // TODO: the blkIdx here is not correct
-        //     tsdbAddSubBlock(pHelper, &compBlock, blkIdx, pHelper->pDataCols[1]->numOfRows);
-        //   }
-        // }
-
-        // TSKEY key1 = iter1 >= pHelper->pDataCols[0]->numOfRows
-        //                  ? INT64_MAX
-        //                  : ((int64_t *)(pHelper->pDataCols[0]->cols[0].pData))[iter1];
-        // TSKEY key2 = iter2 >= rowsWritten ? INT64_MAX : ((int64_t *)(pDataCols->cols[0].pData))[iter2];
-
-        // if (key1 < key2) {
-        //   for (int i = 0; i < pDataCols->numOfCols; i++) {
-        //     SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i;
-        //     memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfRows),
-        //            ((char *)pHelper->pDataCols[0]->cols[i].pData + TYPE_BYTES[pDataCol->type] * iter1),
-        //            TYPE_BYTES[pDataCol->type]);
-        //   }
-        //   pHelper->pDataCols[1]->numOfRows++;
-        //   iter1++;
-        // } else if (key1 == key2) {
-        //   // TODO: think about duplicate key cases
-        //   ASSERT(false);
-        // } else {
-        //   for (int i = 0; i < pDataCols->numOfCols; i++) {
-        //     SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i;
-        //     memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfRows),
-        //            ((char *)pDataCols->cols[i].pData +
-        //             TYPE_BYTES[pDataCol->type] * iter2),
-        //            TYPE_BYTES[pDataCol->type]);
-        //   }
-        //   pHelper->pDataCols[1]->numOfRows++;
-        //   iter2++;
-        // }
-
-        // if (pHelper->pDataCols[0]->numOfRows >= pHelper->config.maxRowsPerFileBlock * 4 / 5) {
-        //   if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows, &compBlock, false, true) < 0) goto _err;
-        //   // TODO: blkIdx here is not correct, fix it
-        //   tsdbInsertSuperBlock(pHelper, &compBlock, blkIdx);
-
-        //   tdResetDataCols(pHelper->pDataCols[1]);
-        // }
       }
     }
   }

src/util/inc/tcache.h

@@ -111,6 +111,15 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz
  */
 void *taosCacheAcquireByName(SCacheObj *pCacheObj, const char *key);
 
+/**
+ * update the expire time of data in cache 
+ * @param pCacheObj     cache object
+ * @param key           key
+ * @param expireTime    new expire time of data
+ * @return
+ */ 
+void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, uint64_t expireTime);
+
 /**
  * Add one reference count for the exist data, and assign this data for a new owner.
  * The new owner needs to invoke the taosCacheRelease when it does not need this data anymore.

src/util/src/tcache.c

@@ -488,6 +488,35 @@ void *taosCacheAcquireByName(SCacheObj *pCacheObj, const char *key) {
   return (ptNode != NULL) ? (*ptNode)->data : NULL;
 }
 
+void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, uint64_t expireTime) {
+  if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) {
+    return NULL;
+  }
+  
+  uint32_t keyLen = (uint32_t)strlen(key);
+  
+  __cache_rd_lock(pCacheObj);
+  
+  SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen);
+  if (ptNode != NULL) {
+     T_REF_INC(*ptNode);
+    (*ptNode)->expiredTime = expireTime;
+  }
+  
+  __cache_unlock(pCacheObj);
+  
+  if (ptNode != NULL) {
+    atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1);
+    uTrace("key:%s expireTime is updated in cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode));
+  } else {
+    atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1);
+    uTrace("key:%s not in cache, retrieved failed", key);
+  }
+  
+  atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1);
+  return (ptNode != NULL) ? (*ptNode)->data : NULL;
+}
+
 void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) {
   if (pCacheObj == NULL || data == NULL) return NULL;
   

tests/pytest/query/filter.py

@@ -87,7 +87,7 @@ class TDTestCase:
 
         # <> for timestamp type
         tdSql.query("select * from db.st where ts <> '2020-05-13 10:00:00.002'")
-        #tdSql.checkRows(4)
+        # tdSql.checkRows(4)
 
         # <> for numeric type
         tdSql.query("select * from db.st where tagtype <> 2")
@@ -96,7 +96,7 @@ class TDTestCase:
         # <> for nchar type
         tdSql.query("select * from db.st where name <> 'first'")
         tdSql.checkRows(4)
-        
+
         # % for nchar type
         tdSql.query("select * from db.st where name like 'fi%'")
         tdSql.checkRows(2)

tests/pytest/query/filterCombo.py

@@ -42,14 +42,17 @@ class TDTestCase:
             ('2020-05-13 10:00:00.005', 3, 'third')""")
 
         # query with filter condition A OR condition B
-        tdSql.query("select * from db.st where ts > '2020-05-13 10:00:00.002' AND tagtype < 2")
+        tdSql.query(
+            "select * from db.st where ts > '2020-05-13 10:00:00.002' AND tagtype < 2")
         tdSql.checkRows(1)
 
         # query with filter condition A OR condition B, error expected
-        tdSql.error("select * from db.st where ts > '2020-05-13 10:00:00.002' OR tagtype < 2")
+        tdSql.error(
+            "select * from db.st where ts > '2020-05-13 10:00:00.002' OR tagtype < 2")
 
         # illegal condition
-        tdSql.error("select * from db.st where ts != '2020-05-13 10:00:00.002' OR tagtype < 2")        
+        tdSql.error(
+            "select * from db.st where ts != '2020-05-13 10:00:00.002' OR tagtype < 2")
         tdSql.error("select * from db.st where tagtype <> 1 OR tagtype < 2")
 
     def stop(self):

tests/pytest/query/queryError.py

@@ -41,18 +41,17 @@ class TDTestCase:
             ('2020-05-13 10:00:00.002', 3, 'third') dev_002 VALUES('2020-05-13 10:00:00.003', 1, 'first'), ('2020-05-13 10:00:00.004', 2, 'second'),
             ('2020-05-13 10:00:00.005', 3, 'third')""")
 
-        
         # query first .. as ..
-        tdSql.error("select first(*) as one from st")    
+        tdSql.error("select first(*) as one from st")
 
         # query last .. as ..
-        tdSql.error("select last(*) as latest from st")        
+        tdSql.error("select last(*) as latest from st")
 
         # query last row .. as ..
-        tdSql.error("select last_row as latest from st") 
+        tdSql.error("select last_row as latest from st")
 
         # query distinct on normal colnum
-        tdSql.error("select distinct tagtype from st")     
+        tdSql.error("select distinct tagtype from st")
 
         # query .. order by non-time field
         tdSql.error("select * from st order by name")

tests/pytest/query/queryNormal.py

@@ -28,17 +28,20 @@ class TDTestCase:
 
         print("==============step1")
 
-        tdSql.execute("create table stb1 (ts timestamp, c1 int, c2 float) tags(t1 int, t2 binary(10), t3 nchar(10))")
-        tdSql.execute("insert into tb1 using stb1 tags(1,'tb1', '表1') values ('2020-04-18 15:00:00.000', 1, 0.1), ('2020-04-18 15:00:01.000', 2, 0.1)")
-        tdSql.execute("insert into tb2 using stb1 tags(2,'tb2', '表2') values ('2020-04-18 15:00:02.000', 3, 2.1), ('2020-04-18 15:00:03.000', 4, 2.2)")        
-        
-        # inner join --- bug 
+        tdSql.execute(
+            "create table stb1 (ts timestamp, c1 int, c2 float) tags(t1 int, t2 binary(10), t3 nchar(10))")
+        tdSql.execute(
+            "insert into tb1 using stb1 tags(1,'tb1', '表1') values ('2020-04-18 15:00:00.000', 1, 0.1), ('2020-04-18 15:00:01.000', 2, 0.1)")
+        tdSql.execute(
+            "insert into tb2 using stb1 tags(2,'tb2', '表2') values ('2020-04-18 15:00:02.000', 3, 2.1), ('2020-04-18 15:00:03.000', 4, 2.2)")
+
+        # inner join --- bug
         tdSql.query("select * from tb1 a, tb2 b where a.ts = b.ts")
         tdSql.checkRows(1)
 
         # join 3 tables -- bug exists
         tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_p.id, stb_p.dscrption, stb_p.pressure,stb_v.velocity from stb_p, stb_t, stb_v where stb_p.ts=stb_t.ts and stb_p.ts=stb_v.ts and stb_p.id = stb_t.id")
-        
+
         # query show stable
         tdSql.query("show stables")
         tdSql.checkRows(1)
@@ -51,15 +54,15 @@ class TDTestCase:
         tdSql.query("select count(*) from stb1")
         tdSql.checkData(0, 0, 4)
 
-        # query first 
+        # query first
         tdSql.query("select first(*) from stb1")
         tdSql.checkData(0, 1, 1)
-        
-        # query last 
+
+        # query last
         tdSql.query("select last(*) from stb1")
         tdSql.checkData(0, 1, 4)
 
-        # query last_row 
+        # query last_row
         tdSql.query("select last_row(*) from stb1")
         tdSql.checkData(0, 1, 4)
 
@@ -70,7 +73,7 @@ class TDTestCase:
         # query first ... as
         tdSql.query("select first(*) as begin from stb1")
         tdSql.checkData(0, 1, 1)
-        
+
         # query last ... as
         tdSql.query("select last(*) as end from stb1")
         tdSql.checkData(0, 1, 4)
@@ -93,7 +96,7 @@ class TDTestCase:
 
         # query ... alias for table ---- bug
         tdSql.query("select t.ts from tb1 t")
-        tdSql.checkRows(2)       
+        tdSql.checkRows(2)
 
         # query ... tbname
         tdSql.query("select tbname from stb1")

tests/pytest/random-test/random-test-multi-threading-3.py

@@ -111,7 +111,6 @@ class Test (threading.Thread):
                 last_tb)
             written = written + 1
 
-
     def drop_stable(self):
         tdLog.info("drop_stable")
         global last_stb
@@ -152,7 +151,6 @@ class Test (threading.Thread):
                 last_tb = ""
                 written = 0
 
-
     def query_data_from_stable(self):
         tdLog.info("query_data_from_stable")
         global last_stb
@@ -164,7 +162,6 @@ class Test (threading.Thread):
             tdLog.info("will query data from super table")
             tdSql.execute('select * from %s' % last_stb)
 
-
     def reset_query_cache(self):
         tdLog.info("reset_query_cache")
         global last_tb
@@ -232,7 +229,7 @@ class Test (threading.Thread):
                 self.threadLock.acquire()
                 tdLog.notice("first thread")
                 randDataOp = random.randint(1, 3)
-                dataOp.get(randDataOp , lambda: "ERROR")()
+                dataOp.get(randDataOp, lambda: "ERROR")()
                 self.threadLock.release()
 
         elif (self.threadId == 2):

tests/pytest/random-test/random-test-multi-threading.py

@@ -111,7 +111,6 @@ class Test (threading.Thread):
                 last_tb)
             written = written + 1
 
-
     def drop_stable(self):
         tdLog.info("drop_stable")
         global last_stb
@@ -154,7 +153,6 @@ class Test (threading.Thread):
                 last_tb = ""
                 written = 0
 
-
     def query_data_from_stable(self):
         tdLog.info("query_data_from_stable")
         global last_stb
@@ -166,7 +164,6 @@ class Test (threading.Thread):
             tdLog.info("will query data from super table")
             tdSql.execute('select * from %s' % last_stb)
 
-
     def reset_query_cache(self):
         tdLog.info("reset_query_cache")
         global last_tb
@@ -230,7 +227,7 @@ class Test (threading.Thread):
                 self.threadLock.acquire()
                 tdLog.notice("first thread")
                 randDataOp = random.randint(1, 3)
-                dataOp.get(randDataOp , lambda: "ERROR")()
+                dataOp.get(randDataOp, lambda: "ERROR")()
                 self.threadLock.release()
 
         elif (self.threadId == 2):

tests/pytest/random-test/random-test.py

@@ -112,7 +112,6 @@ class Test:
             tdSql.execute('drop table %s' % self.last_stb)
             self.last_stb = ""
 
-
     def query_data_from_stable(self):
         tdLog.info("query_data_from_stable")
         if (self.last_stb == ""):
@@ -122,20 +121,21 @@ class Test:
             tdLog.info("will query data from super table")
             tdSql.execute('select * from %s' % self.last_stb)
 
-
     def restart_database(self):
         tdLog.info("restart_databae")
         tdDnodes.stop(1)
         tdDnodes.start(1)
         tdLog.sleep(5)
 
-
     def force_restart_database(self):
         tdLog.info("force_restart_database")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
         tdLog.sleep(5)
         tdSql.prepare()
+        self.last_tb = ""
+        self.last_stb = ""
+        self.written = 0
 
     def drop_table(self):
         tdLog.info("drop_table")
@@ -159,6 +159,9 @@ class Test:
         tdDnodes.start(1)
         tdLog.sleep(5)
         tdSql.prepare()
+        self.last_tb = ""
+        self.last_stb = ""
+        self.written = 0
 
     def delete_datafiles(self):
         tdLog.info("delete_datafiles")
@@ -173,6 +176,9 @@ class Test:
         tdDnodes.start(1)
         tdLog.sleep(10)
         tdSql.prepare()
+        self.last_tb = ""
+        self.last_stb = ""
+        self.written = 0
 
 
 class TDTestCase:

tests/script/unique/arbitrator/dn3_mn1_replica_change.sim

@@ -4,9 +4,9 @@ system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
 system sh/deploy.sh -n dnode4 -i 4
 
-system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -28,11 +28,6 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
 
-system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
-
 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
@@ -94,6 +89,72 @@ if $data00 != $totalRows then
 	return -1
 endi
 
+
+
+
+print ============== step2-1: start dnode2 for falling disc, then restart dnode2, and check rows
+system sh/exec.sh -n dnode2 -s stop
+sleep $sleepTimer
+wait_dnode2_offline_0:
+sql show dnodes
+if $rows != 2 then
+  sleep 2000
+  goto wait_dnode2_offline_0
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+#$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode2Status != offline then
+  sleep 2000
+  goto wait_dnode2_offline_0
+endi
+
+system sh/exec.sh -n dnode2 -s start
+sleep $sleepTimer
+wait_dnode2_reready:
+sql show dnodes
+if $rows != 2 then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+#$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+
+sql select count(*) from $stb
+sleep 1000
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+
+
+
+
+
 print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync
 system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname3