update

include/common/tcommon.h

@@ -56,7 +56,8 @@ typedef enum EStreamType {
   STREAM_CLEAR,
   STREAM_INVALID,
   STREAM_GET_ALL,
-  STREAM_DELETE,
+  STREAM_DELETE_RESULT,
+  STREAM_DELETE_DATA,
   STREAM_RETRIEVE,
   STREAM_PULL_DATA,
   STREAM_PULL_OVER,

include/libs/qcom/query.h

@@ -249,13 +249,6 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
   ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_NODE_NOT_DEPLOYED ||                 \
    (_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_APP_NOT_READY)
 
-#define NEED_SCHEDULER_RETRY_ERROR(_code)                                                         \
-  (NEED_SCHEDULER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL ||            \
-   (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR || (_code) == TSDB_CODE_RPC_BROKEN_LINK)
-  
-
-
-
 #define REQUEST_TOTAL_EXEC_TIMES 2
 
 #define qFatal(...)                                                                           \

include/os/osMath.h

@@ -20,6 +20,12 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following sectio
+#ifndef ALLOW_FORBID_FUNC
+#define qsort  QSORT_FUNC_TAOS_FORBID
+#endif
+
 #define TPOW2(x) ((x) * (x))
 #define TABS(x) ((x) > 0 ? (x) : -(x))
 

include/os/osString.h

@@ -67,7 +67,7 @@ bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs
 int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
 TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
 bool    taosValidateEncodec(const char *encodec);
-int32_t taosHexEncode(const char *src, char *dst, int32_t len);
+int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len);
 int32_t taosHexDecode(const char *src, char *dst, int32_t len);
 
 int32_t taosWcharWidth(TdWchar wchar);

include/util/taoserror.h

@@ -623,6 +623,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_RSMA_INVALID_ENV               TAOS_DEF_ERROR_CODE(0, 0x3150)
 #define TSDB_CODE_RSMA_INVALID_STAT              TAOS_DEF_ERROR_CODE(0, 0x3151)
 #define TSDB_CODE_RSMA_QTASKINFO_CREATE          TAOS_DEF_ERROR_CODE(0, 0x3152)
+#define TSDB_CODE_RSMA_FILE_CORRUPTED            TAOS_DEF_ERROR_CODE(0, 0x3153)
 
 //index
 #define TSDB_CODE_INDEX_REBUILDING               TAOS_DEF_ERROR_CODE(0, 0x3200)

source/client/src/clientImpl.c

@@ -786,6 +786,7 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
 void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
   SRequestObj* pRequest = (SRequestObj*)param;
   pRequest->code = code;
+  pRequest->body.resInfo.execRes = pResult->res;
 
   if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type ||
       TDMT_VND_CREATE_TABLE == pRequest->type) {
@@ -797,6 +798,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
     }
   }
 
+  taosMemoryFree(pResult);
+
   tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code,
            tstrerror(code), pRequest->requestId);
 

source/common/src/tdatablock.c

@@ -1259,6 +1259,7 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
 
   dst->info.rows = src->info.rows;
   dst->info.window = src->info.window;
+  dst->info.type = src->info.type;
   return TSDB_CODE_SUCCESS;
 }
 

source/common/src/ttime.c

@@ -589,7 +589,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec
       return TSDB_CODE_FAILED;
     }
     newColData[len] = 0;
-    int32_t ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, tsDaylight);
+    int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight);
     if (ret != TSDB_CODE_SUCCESS) {
       taosMemoryFree(newColData);
       return ret;

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -21,7 +21,7 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) {
 
   taosThreadRwlockRdlock(&pMgmt->lock);
   taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
-  if (pVnode == NULL) {
+  if (pVnode == NULL || pVnode->dropped) {
     terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
   } else {
     int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
@@ -81,16 +81,18 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
 
   vmReleaseVnode(pMgmt, pVnode);
   while (pVnode->refCount > 0) taosMsleep(10);
+  dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId);
+
   while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
+  dTrace("vgId:%d, vnode-fetch queue is empty", pVnode->vgId);
 
   vmFreeQueue(pMgmt, pVnode);
   vnodeClose(pVnode->pImpl);
   pVnode->pImpl = NULL;
-
   dDebug("vgId:%d, vnode is closed", pVnode->vgId);
 
   if (pVnode->dropped) {

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -146,8 +146,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
-    dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(),
-            TMSG_INFO(pMsg->msgType));
+    dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, msgtype:%s qtype:%d", pHead->vgId, pMsg,
+            terrstr(), TMSG_INFO(pMsg->msgType), qtype);
     return terrno != 0 ? terrno : -1;
   }
 

source/dnode/mnode/impl/inc/mndDef.h

@@ -556,7 +556,7 @@ typedef struct {
   int64_t uid;
   int8_t  status;
   // config
-  int8_t  dropPolicy;
+  int8_t  igExpired;
   int8_t  trigger;
   int64_t triggerParam;
   int64_t watermark;

source/dnode/mnode/impl/inc/mndStream.h

@@ -38,6 +38,8 @@ int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 
+int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/src/mndDb.c

@@ -21,6 +21,7 @@
 #include "mndShow.h"
 #include "mndSma.h"
 #include "mndStb.h"
+#include "mndStream.h"
 #include "mndSubscribe.h"
 #include "mndTopic.h"
 #include "mndTrans.h"
@@ -927,6 +928,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
   if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
+  if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER;
 
@@ -947,7 +949,6 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
   mndTransSetRpcRsp(pTrans, pRsp, rspLen);
 
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
-
   code = 0;
 
 _OVER:

source/dnode/mnode/impl/src/mndDef.c

@@ -28,7 +28,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
   if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
 
-  if (tEncodeI8(pEncoder, pObj->dropPolicy) < 0) return -1;
+  if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->triggerParam) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->watermark) < 0) return -1;
@@ -73,7 +73,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
   if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
 
-  if (tDecodeI8(pDecoder, &pObj->dropPolicy) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->triggerParam) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->watermark) < 0) return -1;

source/dnode/mnode/impl/src/mndSma.c

@@ -523,6 +523,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
   streamObj.updateTime = streamObj.createTime;
   streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
   streamObj.sourceDbUid = pDb->uid;
+  streamObj.targetDbUid = pDb->uid;
   streamObj.version = 1;
   streamObj.sql = pCreate->sql;
   streamObj.smaId = smaObj.uid;
@@ -854,35 +855,25 @@ _OVER:
 
 int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
   SSdb *pSdb = pMnode->pSdb;
-  SSmaObj *pSma = NULL;
   void *pIter = NULL;
-  SVgObj  *pVgroup = NULL;
-  int32_t  code = -1;
 
   while (1) {
+    SSmaObj *pSma = NULL;
     pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma);
     if (pIter == NULL) break;
 
     if (pSma->dbUid == pDb->uid) {
-      pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId);
-      if (pVgroup == NULL) goto _OVER;
-      if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER;
-      if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER;
-      if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
-      mndReleaseVgroup(pMnode, pVgroup);
-      pVgroup = NULL;
+      if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) {
+        sdbRelease(pSdb, pSma);
+        sdbCancelFetch(pSdb, pSma);
+        return -1;
+      }
     }
 
     sdbRelease(pSdb, pSma);
   }
 
-  code = 0;
-
-_OVER:
-  sdbCancelFetch(pSdb, pIter);
-  sdbRelease(pSdb, pSma);
-  mndReleaseVgroup(pMnode, pVgroup);
-  return code;
+  return 0;
 }
 
 static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) {

source/dnode/mnode/impl/src/mndStream.c

@@ -248,7 +248,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
   pObj->status = 0;
 
   // TODO
-  pObj->dropPolicy = 0;
+  pObj->igExpired = pCreate->igExpired;
   pObj->trigger = pCreate->triggerType;
   pObj->triggerParam = pCreate->maxDelay;
   pObj->watermark = pCreate->watermark;
@@ -301,6 +301,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
       .streamQuery = true,
       .triggerType = pObj->trigger == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : pObj->trigger,
       .watermark = pObj->watermark,
+      .igExpired = pObj->igExpired,
   };
 
   // using ast and param to build physical plan
@@ -673,27 +674,29 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
 
 int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
   SSdb *pSdb = pMnode->pSdb;
-
   void *pIter = NULL;
-  SStreamObj *pStream = NULL;
+
   while (1) {
+    SStreamObj *pStream = NULL;
     pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
     if (pIter == NULL) break;
 
     if (pStream->sourceDbUid == pDb->uid || pStream->targetDbUid == pDb->uid) {
       if (pStream->sourceDbUid != pStream->targetDbUid) {
         sdbRelease(pSdb, pStream);
+        sdbCancelFetch(pSdb, pIter);
+        mError("db:%s, failed to drop stream:%s since sourceDbUid:%" PRId64 " not match with targetDbUid:%" PRId64,
+               pDb->name, pStream->name, pStream->sourceDbUid, pStream->targetDbUid);
+        terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST;
         return -1;
       } else {
         // TODO drop all task on snode
         if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) {
           sdbRelease(pSdb, pStream);
+          sdbCancelFetch(pSdb, pIter);
           return -1;
         }
       }
-    } else {
-      sdbRelease(pSdb, pStream);
-      continue;
     }
 
 #if 0

source/dnode/mnode/impl/src/mndTrans.c

@@ -90,7 +90,7 @@ static int32_t mndTransGetActionsSize(SArray *pArray) {
   for (int32_t i = 0; i < actionNum; ++i) {
     STransAction *pAction = taosArrayGet(pArray, i);
     if (pAction->actionType == TRANS_ACTION_RAW) {
-      rawDataLen += (sdbGetRawTotalSize(pAction->pRaw) + sizeof(int32_t));
+      rawDataLen += (sizeof(STransAction) + sdbGetRawTotalSize(pAction->pRaw));
     } else if (pAction->actionType == TRANS_ACTION_MSG) {
       rawDataLen += (sizeof(STransAction) + pAction->contLen);
     } else {
@@ -105,7 +105,7 @@ static int32_t mndTransGetActionsSize(SArray *pArray) {
 static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
-  int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE;
+  int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE + pTrans->paramLen;
   rawDataLen += mndTransGetActionsSize(pTrans->redoActions);
   rawDataLen += mndTransGetActionsSize(pTrans->undoActions);
   rawDataLen += mndTransGetActionsSize(pTrans->commitActions);
@@ -226,7 +226,8 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
 
 _OVER:
   if (terrno != 0) {
-    mError("trans:%d, failed to encode to raw:%p len:%d since %s", pTrans->id, pRaw, dataPos, terrstr());
+    mError("trans:%d, failed to encode to raw:%p maxlen:%d len:%d since %s", pTrans->id, pRaw, sdbGetRawTotalSize(pRaw),
+           dataPos, terrstr());
     sdbFreeRaw(pRaw);
     return NULL;
   }
@@ -1025,7 +1026,7 @@ static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction
   pTrans->lastAction = pAction->id;
   pTrans->lastMsgType = pAction->msgType;
   pTrans->lastEpset = pAction->epSet;
-  pTrans->lastErrorNo == 0;
+  pTrans->lastErrorNo = 0;
   return 0;
 }
 

source/dnode/vnode/CMakeLists.txt

@@ -31,6 +31,8 @@ target_sources(
     "src/sma/smaEnv.c"
     "src/sma/smaUtil.c"
     "src/sma/smaOpen.c"
+    "src/sma/smaCommit.c"
+    "src/sma/smaSnapshot.c"
     "src/sma/smaRollup.c"
     "src/sma/smaTimeRange.c"
 

source/dnode/vnode/src/inc/sma.h

@@ -92,8 +92,9 @@ enum {
   TASK_TRIGGER_STAT_INIT = 0,
   TASK_TRIGGER_STAT_ACTIVE = 1,
   TASK_TRIGGER_STAT_INACTIVE = 2,
-  TASK_TRIGGER_STAT_CANCELLED = 3,
-  TASK_TRIGGER_STAT_FINISHED = 4,
+  TASK_TRIGGER_STAT_PAUSED = 3,
+  TASK_TRIGGER_STAT_CANCELLED = 4,
+  TASK_TRIGGER_STAT_FINISHED = 5,
 };
 void  tdDestroySmaEnv(SSmaEnv *pSmaEnv);
 void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
@@ -214,25 +215,22 @@ struct STFInfo {
 };
 
 struct STFile {
+  uint8_t   state;
   STFInfo   info;
-  STfsFile  f;
+  char     *fname;
   TdFilePtr pFile;
-  uint8_t   state;
 };
 
-#define TD_TFILE_F(tf)            (&((tf)->f))
 #define TD_TFILE_PFILE(tf)        ((tf)->pFile)
 #define TD_TFILE_OPENED(tf)       (TD_TFILE_PFILE(tf) != NULL)
-#define TD_TFILE_FULL_NAME(tf)    (TD_TFILE_F(tf)->aname)
-#define TD_TFILE_REL_NAME(tf)     (TD_TFILE_F(tf)->rname)
+#define TD_TFILE_FULL_NAME(tf)    ((tf)->fname)
 #define TD_TFILE_OPENED(tf)       (TD_TFILE_PFILE(tf) != NULL)
 #define TD_TFILE_CLOSED(tf)       (!TD_TFILE_OPENED(tf))
 #define TD_TFILE_SET_CLOSED(f)    (TD_TFILE_PFILE(f) = NULL)
 #define TD_TFILE_SET_STATE(tf, s) ((tf)->state = (s))
-#define TD_TFILE_DID(tf)          (TD_TFILE_F(tf)->did)
 
-int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname);
-int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType);
+int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname);
+int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType);
 int32_t tdOpenTFile(STFile *pTFile, int flags);
 int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte);
 int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence);
@@ -244,8 +242,10 @@ int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo);
 int32_t tdUpdateTFileHeader(STFile *pTFile);
 void    tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
 void    tdCloseTFile(STFile *pTFile);
+void    tdDestroyTFile(STFile *pTFile);
 
-void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName);
+void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, int64_t version, char *outputName);
+void tdGetVndDirName(int32_t vgId, const char *dname, char *outputName);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -165,6 +165,9 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
 int32_t smaOpen(SVnode* pVnode);
 int32_t smaCloseEnv(SSma* pSma);
 int32_t smaCloseEx(SSma* pSma);
+int32_t smaPreCommit(SSma* pSma);
+int32_t smaCommit(SSma* pSma);
+int32_t smaPostCommit(SSma* pSma);
 
 int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
 int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);

source/dnode/vnode/src/sma/smaCommit.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sma.h"
+
+static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma);
+static int32_t tdProcessRSmaCommitImpl(SSma *pSma);
+static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma);
+
+/**
+ * @brief Only applicable to Rollup SMA
+ *
+ * @param pSma
+ * @return int32_t
+ */
+int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); }
+
+/**
+ * @brief Only applicable to Rollup SMA
+ *
+ * @param pSma
+ * @return int32_t
+ */
+int32_t smaCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); }
+
+/**
+ * @brief Only applicable to Rollup SMA
+ *
+ * @param pSma
+ * @return int32_t
+ */
+int32_t smaPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); }
+
+/**
+ * @brief pre-commit for rollup sma.
+ *  1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED.
+ *  2) perform persist task for qTaskInfo
+ *  3) wait all triggered fetch tasks finished
+ *  4) set trigger stat of rsma timer TASK_TRIGGER_STAT_ACTIVE.
+ *  5) finish
+ *
+ * @param pSma
+ * @return int32_t
+ */
+static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) {
+  SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
+  if (!pSmaEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+
+  // step 1
+  atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED);
+
+  // step 2
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief commit for rollup sma
+ *
+ * @param pSma
+ * @return int32_t
+ */
+static int32_t tdProcessRSmaCommitImpl(SSma *pSma) {
+  SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
+  if (!pSmaEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief post-commit for rollup sma
+ *  1) clean up the outdated qtaskinfo files
+ *
+ * @param pSma
+ * @return int32_t
+ */
+static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) {
+  SVnode *pVnode = pSma->pVnode;
+
+  if (!VND_IS_RSMA(pVnode)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int64_t       committed = pVnode->state.committed;
+  TdDirPtr      pDir = NULL;
+  TdDirEntryPtr pDirEntry = NULL;
+  char          dir[TSDB_FILENAME_LEN];
+  char          bname[TSDB_FILENAME_LEN];
+  const char   *pattern = "^v[0-9]+qtaskinfo\\.ver([0-9]+)?$";
+  regex_t       regex;
+
+  tdGetVndDirName(TD_VID(pVnode), VNODE_RSMA_DIR, dir);
+
+  // Resource allocation and init
+  regcomp(&regex, pattern, REG_EXTENDED);
+
+  if ((pDir = taosOpenDir(dir)) == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    smaWarn("rsma post-commit open dir %s failed since %s", dir, terrstr());
+    return TSDB_CODE_FAILED;
+  }
+
+  regmatch_t regMatch[2];
+  while ((pDirEntry = taosReadDir(pDir)) != NULL) {
+    char *entryName = taosGetDirEntryName(pDirEntry);
+    if (!entryName) {
+      continue;
+    }
+    char *fileName = taosDirEntryBaseName(entryName);
+    int   code = regexec(&regex, bname, 2, regMatch, 0);
+
+    if (code == 0) {
+      // match
+      printf("match 0 = %s\n", (char *)POINTER_SHIFT(fileName, regMatch[0].rm_so));
+      printf("match 1 = %s\n", (char *)POINTER_SHIFT(fileName, regMatch[1].rm_so));
+    } else if (code == REG_NOMATCH) {
+      // not match
+      smaInfo("rsma post-commit not match %s", fileName);
+      continue;
+    } else {
+      // has other error
+      terrno = TAOS_SYSTEM_ERROR(code);
+      smaWarn("rsma post-commit regexec failed since %s", terrstr());
+
+      taosCloseDir(&pDir);
+      regfree(&regex);
+      return TSDB_CODE_FAILED;
+    }
+  }
+  taosCloseDir(&pDir);
+  return TSDB_CODE_SUCCESS;
+}

source/dnode/vnode/src/sma/smaRollup.c

@@ -23,8 +23,7 @@ SSmaMgmt smaMgmt = {
     .smaRef = -1,
 };
 
-typedef enum { TD_QTASK_TMP_F = 0, TD_QTASK_CUR_F } TD_QTASK_FILE_T;
-static const char                *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
+#define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver"
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
 
@@ -37,7 +36,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
 static void    tdRSmaFetchTrigger(void *param, void *tmrId);
 static void    tdRSmaPersistTrigger(void *param, void *tmrId);
 static void   *tdRSmaPersistExec(void *param);
-static void    tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName);
+static void    tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);
 
 static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile);
 static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish);
@@ -88,8 +87,8 @@ struct SRSmaQTaskInfoIter {
   int32_t nBufPos;
 };
 
-static void tdRSmaQTaskInfoGetFName(int32_t vgId, int8_t ftype, char *outputName) {
-  tdGetVndFileName(vgId, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName);
+static void tdRSmaQTaskInfoGetFName(int32_t vgId, int64_t version, char *outputName) {
+  tdGetVndFileName(vgId, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName);
 }
 
 static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) {
@@ -493,7 +492,6 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) {
 }
 
 static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
-  ASSERT(pMsg != NULL);
   SSubmitMsgIter msgIter = {0};
   SSubmitBlk    *pBlock = NULL;
   SSubmitBlkIter blkIter = {0};
@@ -501,19 +499,26 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
 
   terrno = TSDB_CODE_SUCCESS;
 
-  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
+  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) {
+    return -1;
+  }
   while (true) {
-    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
+    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) {
+      return -1;
+    }
 
     if (!pBlock) break;
     tdUidStorePut(pStore, msgIter.suid, NULL);
   }
 
-  if (terrno != TSDB_CODE_SUCCESS) return -1;
+  if (terrno != TSDB_CODE_SUCCESS) {
+    return -1;
+  }
   return 0;
 }
 
 static void tdDestroySDataBlockArray(SArray *pArray) {
+  // TODO
 #if 0
   for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
     SSDataBlock *pDataBlock = taosArrayGet(pArray, i);
@@ -598,18 +603,25 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
 
   pSma = RSMA_INFO_SMA(pItem->pRsmaInfo);
 
-  // if rsma trigger stat in cancelled or finished, not start fetch task anymore
+  // if rsma trigger stat in paused, cancelled or finished, not start fetch task
   int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
-  if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) {
+  switch (rsmaTriggerStat) {
+    case TASK_TRIGGER_STAT_PAUSED:
+    case TASK_TRIGGER_STAT_CANCELLED:
+    case TASK_TRIGGER_STAT_FINISHED: {
       taosReleaseRef(smaMgmt.smaRef, pItem->refId);
       smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled",
                SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
       return;
     }
+    default:
+      break;
+  }
 
   int8_t fetchTriggerStat =
       atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
-  if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) {
+  switch (fetchTriggerStat) {
+    case TASK_TRIGGER_STAT_ACTIVE: {
       smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
                pItem->level, pItem->pRsmaInfo->suid);
 
@@ -620,11 +632,25 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
       tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
 
       tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
-
-  } else {
+    } break;
+    case TASK_TRIGGER_STAT_PAUSED: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
+               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INACTIVE: {
       smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
                SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INIT: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
+               pItem->level, pItem->pRsmaInfo->suid);
+    } break;
+    default: {
+      smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
+              SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+    } break;
   }
+
 _end:
   taosReleaseRef(smaMgmt.smaRef, pItem->refId);
 }
@@ -780,10 +806,10 @@ _err:
 static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
   SVnode *pVnode = pSma->pVnode;
   STFile  tFile = {0};
-  char    qTaskInfoFName[TSDB_FILENAME_LEN];
+  char    qTaskInfoFName[TSDB_FILENAME_LEN] = {0};
 
-  tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_TMP_F, qTaskInfoFName);
-  if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) {
+  tdRSmaQTaskInfoGetFName(TD_VID(pVnode), pVnode->state.committed, qTaskInfoFName);
+  if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
     goto _err;
   }
 
@@ -799,17 +825,20 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
   if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
     tdRSmaQTaskInfoIterDestroy(&fIter);
     tdCloseTFile(&tFile);
+    tdDestroyTFile(&tFile);
     goto _err;
   }
 
   if (tdRSmaQTaskInfoRestore(pSma, &fIter) < 0) {
     tdRSmaQTaskInfoIterDestroy(&fIter);
     tdCloseTFile(&tFile);
+    tdDestroyTFile(&tFile);
     goto _err;
   }
 
   tdRSmaQTaskInfoIterDestroy(&fIter);
   tdCloseTFile(&tFile);
+  tdDestroyTFile(&tFile);
   return TSDB_CODE_SUCCESS;
 _err:
   smaError("rsma restore, qtaskinfo reload failed since %s", terrstr());
@@ -931,19 +960,21 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
   }
 
   if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) {
-    ASSERT(0);
     return TSDB_CODE_FAILED;
   }
 
   if (tdReadTFile(pTFile, pIter->qBuf, nBytes) != nBytes) {
-    ASSERT(0);
     return TSDB_CODE_FAILED;
   }
 
   int32_t infoLen = 0;
   taosDecodeFixedI32(pIter->qBuf, &infoLen);
   if (infoLen > nBytes) {
-    ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE);
+    if (infoLen <= RSMA_QTASKINFO_BUFSIZE) {
+      terrno = TSDB_CODE_RSMA_FILE_CORRUPTED;
+      smaError("iterate rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr());
+      return TSDB_CODE_FAILED;
+    }
     pIter->nAlloc = infoLen;
     void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen);
     if (!pBuf) {
@@ -955,12 +986,10 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
     nBytes = infoLen;
 
     if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) {
-      ASSERT(0);
       return TSDB_CODE_FAILED;
     }
 
     if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) {
-      ASSERT(0);
       return TSDB_CODE_FAILED;
     }
   }
@@ -977,7 +1006,6 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
     // block iter
     bool isFinish = false;
     if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) {
-      ASSERT(0);
       return TSDB_CODE_FAILED;
     }
     if (isFinish) {
@@ -989,6 +1017,7 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
     pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead);
     if (qTaskInfoLenWithHead < RSMA_QTASKINFO_HEAD_LEN) {
       terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+      smaError("restore rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr());
       return TSDB_CODE_FAILED;
     }
 
@@ -1025,22 +1054,16 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
   return TSDB_CODE_SUCCESS;
 }
 
-static void *tdRSmaPersistExec(void *param) {
-  setThreadName("rsma-task-persist");
-  SRSmaStat *pRSmaStat = param;
+static int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) {
   SSma   *pSma = pRSmaStat->pSma;
-  STfs      *pTfs = pSma->pVnode->pTfs;
+  SVnode *pVnode = pSma->pVnode;
   int32_t vid = SMA_VID(pSma);
   int64_t toffset = 0;
   bool    isFileCreated = false;
 
-  if (TASK_TRIGGER_STAT_CANCELLED == atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))) {
-    goto _end;
-  }
-
   void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL);
   if (!infoHash) {
-    goto _end;
+    return TSDB_CODE_SUCCESS;
   }
 
   STFile tFile = {0};
@@ -1074,9 +1097,13 @@ static void *tdRSmaPersistExec(void *param) {
 
       if (!isFileCreated) {
         char qTaskInfoFName[TSDB_FILENAME_LEN];
-        tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_F, qTaskInfoFName);
-        tdInitTFile(&tFile, pTfs, qTaskInfoFName);
-        tdCreateTFile(&tFile, pTfs, true, -1);
+        tdRSmaQTaskInfoGetFName(vid, pSma->pVnode->state.applied, qTaskInfoFName);
+        if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
+          goto _err;
+        }
+        if (tdCreateTFile(&tFile, true, -1) < 0) {
+          goto _err;
+        }
 
         isFileCreated = true;
       }
@@ -1101,49 +1128,55 @@ static void *tdRSmaPersistExec(void *param) {
 
     infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash);
   }
-_normal:
+
   if (isFileCreated) {
     if (tdUpdateTFileHeader(&tFile) < 0) {
       smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile),
                tstrerror(terrno));
-      tdCloseTFile(&tFile);
-      tdRemoveTFile(&tFile);
       goto _err;
     } else {
       smaDebug("vgId:%d, rsma, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile));
     }
 
     tdCloseTFile(&tFile);
-
-    char newFName[TSDB_FILENAME_LEN];
-    strncpy(newFName, TD_TFILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN);
-    char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_F]);
-    strncpy(pos, tdQTaskInfoFname[TD_QTASK_TMP_F], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
-    if (taosRenameFile(TD_TFILE_FULL_NAME(&tFile), newFName) != 0) {
-      smaError("vgId:%d, rsma, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
-      goto _err;
-    } else {
-      smaDebug("vgId:%d, rsma, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
-    }
+    tdDestroyTFile(&tFile);
   }
-  goto _end;
+  return TSDB_CODE_SUCCESS;
 _err:
   if (isFileCreated) {
     tdRemoveTFile(&tFile);
+    tdDestroyTFile(&tFile);
   }
+  return TSDB_CODE_FAILED;
+}
+
+static void *tdRSmaPersistExec(void *param) {
+  setThreadName("rsma-task-persist");
+  SRSmaStat *pRSmaStat = param;
+  SSma      *pSma = pRSmaStat->pSma;
+
+  int8_t triggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat));
+
+  if (TASK_TRIGGER_STAT_CANCELLED == triggerStat || TASK_TRIGGER_STAT_PAUSED == triggerStat) {
+    goto _end;
+  }
+
+  // execution
+  tdRSmaPersistExecImpl(pRSmaStat);
+
 _end:
   if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
                                                                   TASK_TRIGGER_STAT_INACTIVE,
                                                                   TASK_TRIGGER_STAT_ACTIVE)) {
-    smaDebug("vgId:%d, rsma persist task is active again", vid);
+    smaDebug("vgId:%d, rsma persist task is active again", SMA_VID(pSma));
   } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
                                                                           TASK_TRIGGER_STAT_CANCELLED,
                                                                           TASK_TRIGGER_STAT_FINISHED)) {
-    smaDebug("vgId:%d, rsma persist task is cancelled", vid);
+    smaDebug("vgId:%d, rsma persist task is cancelled", SMA_VID(pSma));
   } else {
-    smaWarn("vgId:%d, rsma persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
-    ASSERT(0);
+    smaWarn("vgId:%d, rsma persist task in stat %" PRIi8, SMA_VID(pSma), atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
   }
+
   atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
   taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
   taosThreadExit(NULL);
@@ -1166,8 +1199,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
                                                                             TASK_TRIGGER_STAT_FINISHED)) {
       smaDebug("vgId:%d, persist task is cancelled and set finished", SMA_VID(pRSmaStat->pSma));
     } else {
-      smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)),
-              SMA_VID(pRSmaStat->pSma));
+      smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, SMA_VID(pRSmaStat->pSma),
+              atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
       ASSERT(0);
     }
     atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
@@ -1216,6 +1249,9 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
       atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED);
       smaDebug("rsma persistence not start since cancelled and finished");
     } break;
+    case TASK_TRIGGER_STAT_PAUSED: {
+      smaDebug("rsma persistence not start since paused");
+    } break;
     case TASK_TRIGGER_STAT_INACTIVE: {
       smaDebug("rsma persistence not start since inactive");
     } break;

source/dnode/vnode/src/sma/smaSnapshot.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sma.h"
\ No newline at end of file

source/dnode/vnode/src/sma/smaUtil.c

@@ -179,52 +179,56 @@ void tdCloseTFile(STFile *pTFile) {
   }
 }
 
-void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName) {
-  snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vgId, dname, fname);
+void tdDestroyTFile(STFile *pTFile) { taosMemoryFreeClear(TD_TFILE_FULL_NAME(pTFile)); }
+
+void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, int64_t version, char *outputName) {
+  if (version < 0) {
+    snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/v%d%s", vgId, dname, vgId, fname);
+  } else {
+    snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/v%d%s%" PRIi64, vgId, dname, vgId, fname, version);
+  }
 }
 
-int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) {
-  char    fullname[TSDB_FILENAME_LEN];
-  SDiskID did = {0};
+void tdGetVndDirName(int32_t vgId, const char *dname, char *outputName) {
+  snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", vgId, dname);
+}
 
+int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname) {
   TD_TFILE_SET_STATE(pTFile, TD_FILE_STATE_OK);
   TD_TFILE_SET_CLOSED(pTFile);
 
   memset(&(pTFile->info), 0, sizeof(pTFile->info));
   pTFile->info.magic = TD_FILE_INIT_MAGIC;
 
-  if (tfsAllocDisk(pTfs, 0, &did) < 0) {
-    terrno = TSDB_CODE_NO_AVAIL_DISK;
+  char tmpName[TSDB_FILENAME_LEN * 2 + 32] = {0};
+  snprintf(tmpName, TSDB_FILENAME_LEN * 2 + 32, "%s%s%s", dname, TD_DIRSEP, fname);
+  int32_t tmpNameLen = strlen(tmpName) + 1;
+  pTFile->fname = taosMemoryMalloc(tmpNameLen);
+  if (!pTFile->fname) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
-
-  tfsInitFile(pTfs, &(pTFile->f), did, fname);
+  tstrncpy(pTFile->fname, tmpName, tmpNameLen);
 
   return 0;
 }
 
-int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType) {
+int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType) {
   ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC);
 
   pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pTFile->pFile == NULL) {
     if (errno == ENOENT) {
       // Try to create directory recursively
-      char *s = strdup(TD_TFILE_REL_NAME(pTFile));
-      if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_TFILE_DID(pTFile)) < 0) {
-        taosMemoryFreeClear(s);
+      if (taosMulMkDir(taosDirName(TD_TFILE_FULL_NAME(pTFile))) != 0) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;
-      }
-      taosMemoryFreeClear(s);
-
+      } else {
         pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
         if (pTFile->pFile == NULL) {
           terrno = TAOS_SYSTEM_ERROR(errno);
           return -1;
         }
-    } else {
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      return -1;
       }
     }
 
@@ -240,11 +244,18 @@ int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fTyp
       tdRemoveTFile(pTFile);
       return -1;
     }
+  }
 
   return 0;
 }
 
-int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_TFILE_F(pTFile)); }
+int32_t tdRemoveTFile(STFile *pTFile) {
+  if (taosRemoveFile(TD_TFILE_FULL_NAME(pTFile)) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  };
+  return 0;
+}
 
 // smaXXXUtil ================
 // ...
\ No newline at end of file

source/dnode/vnode/src/tq/tq.c

@@ -183,13 +183,15 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
   } else {
     ASSERT(0);
   }
-  STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);
-  if (pOffset == NULL || pOffset->val.version < offset.val.version) {
+  /*STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);*/
+  /*if (pOffset != NULL) {*/
+  /*if (pOffset->val.type == TMQ_OFFSET__LOG && pOffset->val.version < offset.val.version) {*/
   if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) {
     ASSERT(0);
     return -1;
   }
-  }
+  /*}*/
+  /*}*/
 
   return 0;
 }
@@ -375,8 +377,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
     taosMemoryFree(pCkHead);
   } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    tqInfo("retrieve using snapshot req offset: uid %ld ts %ld, actual offset: uid %ld ts %ld", dataRsp.reqOffset.uid,
-           dataRsp.reqOffset.ts, fetchOffsetNew.uid, fetchOffsetNew.ts);
+    tqInfo("retrieve using snapshot actual offset: uid %ld ts %ld", fetchOffsetNew.uid, fetchOffsetNew.ts);
     if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) {
       ASSERT(0);
     }

source/dnode/vnode/src/tq/tqRead.c

@@ -120,7 +120,9 @@ bool tqNextDataBlock(SStreamReader* pHandle) {
       return true;
     }
     void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->msgIter.uid, sizeof(int64_t));
+    /*tqDebug("search uid %ld", pHandle->msgIter.uid);*/
     if (ret != NULL) {
+      /*tqDebug("find   uid %ld", pHandle->msgIter.uid);*/
       return true;
     }
   }

source/dnode/vnode/src/tq/tqSink.c

@@ -185,5 +185,7 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
       .contLen = ntohl(pReq->length),
   };
 
-  ASSERT(tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) == 0);
+  if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
+    tqDebug("failed to put into write-queue since %s", terrstr());
+  }
 }

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -28,12 +28,12 @@ const SVnodeCfg vnodeCfgDefault = {
                 .update = 1,
                 .compression = 2,
                 .slLevel = 5,
-                .days = 10,
+                .days = 14400,
                 .minRows = 100,
                 .maxRows = 4096,
-                .keep2 = 3650,
-                .keep0 = 3650,
-                .keep1 = 3650},
+                .keep2 = 5256000,
+                .keep0 = 5256000,
+                .keep1 = 5256000},
     .walCfg =
         {.vgId = -1, .fsyncPeriod = 0, .retentionPeriod = 0, .rollPeriod = 0, .segSize = 0, .level = TAOS_WAL_WRITE},
     .hashBegin = 0,

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -229,6 +229,9 @@ int vnodeCommit(SVnode *pVnode) {
     return -1;
   }
 
+  // preCommit
+  // TODO
+  
   // commit each sub-system
   if (metaCommit(pVnode->pMeta) < 0) {
     ASSERT(0);
@@ -269,6 +272,9 @@ int vnodeCommit(SVnode *pVnode) {
   
   pVnode->state.committed = info.state.committed;
 
+  // postCommit
+  smaPostCommit(pVnode->pSma);
+
   // apply the commit (TODO)
   vnodeBufPoolReset(pVnode->onCommit);
   pVnode->onCommit->next = pVnode->pPool;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -779,6 +779,7 @@ _exit:
   taosArrayDestroy(submitRsp.pArray);
 
   // TODO: the partial success scenario and the error case
+  // => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level 1/level 2.
   // TODO: refactor
   if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
     tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);

source/libs/executor/inc/executorimpl.h

@@ -356,6 +356,7 @@ typedef struct SStreamBlockScanInfo {
   SUpdateInfo*    pUpdateInfo;
 
   EStreamScanMode scanMode;
+  SOperatorInfo* pStreamScanOp;
   SOperatorInfo* pSnapshotReadOp;
   SArray*        childIds;
   SessionWindowSupporter sessionSup;
@@ -427,7 +428,7 @@ typedef struct SIntervalAggOperatorInfo {
   STimeWindowAggSupp twAggSup;
   bool               invertible;
   SArray*            pPrevValues;        //  SArray<SGroupKeys> used to keep the previous not null value for interpolation.
-  bool               ignoreCloseWindow;
+  bool               ignoreExpiredData;
 } SIntervalAggOperatorInfo;
 
 typedef struct SStreamFinalIntervalOperatorInfo {
@@ -449,7 +450,7 @@ typedef struct SStreamFinalIntervalOperatorInfo {
   SArray*            pPullWins;          // SPullWindowInfo
   int32_t            pullIndex;
   SSDataBlock*       pPullDataRes;
-  bool               ignoreCloseWindow;
+  bool               ignoreExpiredData;
 } SStreamFinalIntervalOperatorInfo;
 
 typedef struct SAggOperatorInfo {
@@ -587,7 +588,7 @@ typedef struct SStreamSessionAggOperatorInfo {
   SArray*              pChildren;       // cache for children's result; final stream operator
   SPhysiNode*          pPhyNode;           // create new child
   bool                 isFinal;
-  bool                 ignoreCloseWindow;
+  bool                 ignoreExpiredData;
 } SStreamSessionAggOperatorInfo;
 
 typedef struct STimeSliceOperatorInfo {
@@ -631,7 +632,7 @@ typedef struct SStreamStateAggOperatorInfo {
   void*                pDelIterator;
   SArray*              pScanWindow;
   SArray*              pChildren;       // cache for children's result;
-  bool                 ignoreCloseWindow;
+  bool                 ignoreExpiredData;
 } SStreamStateAggOperatorInfo;
 
 typedef struct SSortedMergeOperatorInfo {

source/libs/executor/src/executor.c

@@ -145,10 +145,12 @@ static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo,
       continue;
     }
 
+    // TODO handle ntb case
     if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) {
       continue;
     }
-    // TODO handle ntb case
+    /*pScanInfo->pStreamScanOp->pTaskInfo->tableqinfoList.*/
+    // handle multiple partition
 
     taosArrayPush(qa, id);
   }

source/libs/executor/src/executorimpl.c

@@ -2027,8 +2027,9 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
 
   ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY);
 
-  qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, GET_TASKID(pTaskInfo),
-         pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, sourceIndex, totalSources);
+  qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu,
+         GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId,
+         sourceIndex, totalSources);
 
   pMsg->header.vgId = htonl(pSource->addr.nodeId);
   pMsg->sId = htobe64(pSource->schedId);
@@ -2163,8 +2164,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
       SSDataBlock*         pRes = pExchangeInfo->pResult;
       SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
       if (pRsp->numOfRows == 0) {
-        qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
-               ", completed:%d try next %d/%" PRIzu,
+        qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64
+               ", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu,
                GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows,
                pExchangeInfo->loadInfo.totalRows, completed + 1, i + 1, totalSources);
         pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
@@ -2183,18 +2184,19 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
       }
 
       if (pRsp->completed == 1) {
-        qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d"
+        qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64
+               " execId:%d"
                " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64
                ", completed:%d try next %d/%" PRIzu,
-               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows, pDataInfo->totalRows,
-               pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources);
+               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows,
+               pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources);
         completed += 1;
         pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
       } else {
         qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
                ", totalBytes:%" PRIu64,
-               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows,
-               pLoadInfo->totalSize);
+               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
+               pLoadInfo->totalRows, pLoadInfo->totalSize);
       }
 
       taosMemoryFreeClear(pDataInfo->pRsp);
@@ -2267,8 +2269,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
 
     if (pDataInfo->code != TSDB_CODE_SUCCESS) {
-      qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), pSource->addr.nodeId,
-             pSource->taskId, pSource->execId, tstrerror(pDataInfo->code));
+      qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo),
+             pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code));
       pOperator->pTaskInfo->code = pDataInfo->code;
       return NULL;
     }
@@ -2276,8 +2278,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     SRetrieveTableRsp*   pRsp = pDataInfo->pRsp;
     SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
     if (pRsp->numOfRows == 0) {
-      qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
-             " try next",
+      qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64
+             ", totalRows:%" PRIu64 " try next",
              GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1,
              pDataInfo->totalRows, pLoadInfo->totalRows);
 
@@ -2296,16 +2298,17 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     if (pRsp->completed == 1) {
       qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64
              ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
-             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pDataInfo->totalRows,
-             pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources);
+             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
+             pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1,
+             totalSources);
 
       pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
       pExchangeInfo->current += 1;
     } else {
       qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
              ", totalBytes:%" PRIu64,
-             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows,
-             pLoadInfo->totalSize);
+             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
+             pLoadInfo->totalRows, pLoadInfo->totalSize);
     }
 
     pOperator->resultInfo.totalRows += pRes->info.rows;
@@ -3250,6 +3253,10 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       doSetOperatorCompleted(pOperator);
       break;
     }
+    if (pBlock->info.type == STREAM_RETRIEVE) {
+      // for stream interval
+      return pBlock;
+    }
 
     // the pDataBlock are always the same one, no need to call this again
     int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);

source/libs/executor/src/scanoperator.c

@@ -807,6 +807,23 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
   return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
 }
 
+static void setGroupId(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
+  ASSERT(rowIndex < pBlock->info.rows);
+  switch (pBlock->info.type)
+  {
+  case STREAM_RETRIEVE: {
+    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
+    uint64_t* groupCol = (uint64_t*)pColInfo->pData;
+    pInfo->groupId = groupCol[rowIndex];
+  }
+    break;
+  case STREAM_DELETE_DATA:
+    break;
+  default:
+    break;
+  }
+}
+
 static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
   STimeWindow win = {
       .skey = INT64_MIN,
@@ -829,6 +846,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3
     } else {
       win =
           getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL);
+      setGroupId(pInfo, pSDB, 2, *pRowIndex);
       (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL,
                                                TSDB_ORDER_ASC);
     }
@@ -1031,10 +1049,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
       }
       pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
     } else {
-      if (isStateWindow(pInfo) && taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) {
+      if (isStateWindow(pInfo)) {
         pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
         pInfo->updateResIndex = pInfo->pUpdateRes->info.rows;
-        prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
+        if (!prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex)) {
+          pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
+        }
       }
       if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) {
         SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
@@ -1274,6 +1294,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
   pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1};
   pInfo->groupId = 0;
   pInfo->pPullDataRes = createPullDataBlock();
+  pInfo->pStreamScanOp = pOperator;
 
   pOperator->name = "StreamBlockScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;

source/libs/executor/src/timewindowoperator.c

@@ -838,7 +838,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
   STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
                                         pInfo->interval.precision, &pInfo->win);
   int32_t     ret = TSDB_CODE_SUCCESS;
-  if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
+  if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) {
     ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                  numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
     if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
@@ -871,7 +871,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
     doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
   }
 
-  if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
+  if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) {
     updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
     doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
                      pBlock->info.rows, numOfOutput, pInfo->order);
@@ -886,7 +886,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
     if (startPos < 0) {
       break;
     }
-    if (pInfo->ignoreCloseWindow && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
+    if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
       ekey = ascScan ? nextWin.ekey : nextWin.skey;
       forwardRows =
           getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
@@ -1535,7 +1535,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
   pInfo->interval = *pInterval;
   pInfo->execModel = pTaskInfo->execModel;
   pInfo->twAggSup = *pTwAggSupp;
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pPhyNode->window.igExpired;
 
   if (pPhyNode->window.pExprs != NULL) {
     int32_t    numOfScalar = 0;
@@ -2292,7 +2292,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
                                             pInfo->interval.precision, NULL);
   while (1) {
     bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
-    if (pInfo->ignoreCloseWindow && isClosed) {
+    if (pInfo->ignoreExpiredData && isClosed) {
       startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
       if (startPos < 0) {
         break;
@@ -2710,7 +2710,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
   pInfo->pPullDataRes = createPullDataBlock();
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
 
   pOperator->operatorType = pPhyNode->type;
   pOperator->blocking = true;
@@ -2852,12 +2852,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
   pInfo->pDelIterator = NULL;
   pInfo->pDelRes = createOneDataBlock(pResBlock, false);
-  pInfo->pDelRes->info.type = STREAM_DELETE;
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
   blockDataEnsureCapacity(pInfo->pDelRes, 64);
   pInfo->pChildren = NULL;
   pInfo->isFinal = false;
   pInfo->pPhyNode = pPhyNode;
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
 
   pOperator->name = "StreamSessionWindowAggOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
@@ -3133,7 +3133,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
 
   SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
   for (int32_t i = 0; i < pSDataBlock->info.rows;) {
-    if (pInfo->ignoreCloseWindow && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
+    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
       i++;
       continue;
     }
@@ -3413,8 +3413,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
   pOperator->status = OP_RES_TO_RETURN;
 
   closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
-      getResWinForSession, pInfo->ignoreCloseWindow);
-  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
+      getResWinForSession, pInfo->ignoreExpiredData);
+  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
   copyUpdateResult(pStUpdated, pUpdated);
   taosHashCleanup(pStUpdated);
 
@@ -3822,7 +3822,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
   SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
   SColumnInfoData*     pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
   for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
-    if (pInfo->ignoreCloseWindow && isOverdue(tsCols[i], &pInfo->twAggSup)) {
+    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
       i++;
       continue;
     }
@@ -3866,12 +3866,14 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_RES_TO_RETURN) {
     doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
     if (pInfo->pDelRes->info.rows > 0) {
+      printDataBlock(pInfo->pDelRes, "single state");
       return pInfo->pDelRes;
     }
     doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
     if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
     }
+    printDataBlock(pBInfo->pRes, "single state");
     return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
   }
 
@@ -3884,6 +3886,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
     if (pBlock == NULL) {
       break;
     }
+    printDataBlock(pBlock, "single state recv");
 
     if (pBlock->info.type == STREAM_CLEAR) {
       doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
@@ -3903,8 +3906,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   pOperator->status = OP_RES_TO_RETURN;
 
   closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
-      getResWinForState, pInfo->ignoreCloseWindow);
-  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
+      getResWinForState, pInfo->ignoreExpiredData);
+  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
   copyUpdateResult(pSeUpdated, pUpdated);
   taosHashCleanup(pSeUpdated);
 
@@ -3914,9 +3917,11 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
   doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
   if (pInfo->pDelRes->info.rows > 0) {
+    printDataBlock(pInfo->pDelRes, "single state");
     return pInfo->pDelRes;
   }
   doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
+  printDataBlock(pBInfo->pRes, "single state");
   return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
 }
 
@@ -3975,10 +3980,10 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
   pInfo->pDelIterator = NULL;
   pInfo->pDelRes = createOneDataBlock(pResBlock, false);
-  pInfo->pDelRes->info.type = STREAM_DELETE;
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
   blockDataEnsureCapacity(pInfo->pDelRes, 64);
   pInfo->pChildren = NULL;
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pStateNode->window.igExpired;
 
   pOperator->name = "StreamStateAggOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;

source/libs/function/src/builtins.c

@@ -39,6 +39,174 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char
   return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName);
 }
 
+#define TIME_UNIT_INVALID 1
+#define TIME_UNIT_TOO_SMALL 2
+
+static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
+  if (!pVal->isDuration) {
+    return TIME_UNIT_INVALID;
+  }
+
+  if (TSDB_TIME_PRECISION_MILLI == dbPrec && 0 == strcasecmp(pVal->literal, "1u")) {
+    return TIME_UNIT_TOO_SMALL;
+  }
+
+  if (pVal->literal[0] != '1' || (pVal->literal[1] != 'u' && pVal->literal[1] != 'a' &&
+                                  pVal->literal[1] != 's' && pVal->literal[1] != 'm' &&
+                                  pVal->literal[1] != 'h' && pVal->literal[1] != 'd' &&
+                                  pVal->literal[1] != 'w')) {
+    return TIME_UNIT_INVALID;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/* Following are valid ISO-8601 timezone format:
+ * 1 z/Z
+ * 2 ±hh:mm
+ * 3 ±hhmm
+ * 4 ±hh
+ *
+ */
+
+static bool validateHourRange(int8_t hour) {
+  if (hour < 0 || hour > 12) {
+    return false;
+  }
+
+  return true;
+}
+
+static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
+  if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '+')) {
+    return true;
+  }
+
+  return false;
+}
+
+static bool validateTimestampDigits(const SValueNode* pVal) {
+  if (!IS_INTEGER_TYPE(pVal->node.resType.type)) {
+    return false;
+  }
+
+  int64_t tsVal = pVal->datum.i;
+  char    fraction[20] = {0};
+  NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction);
+  int32_t tsDigits = (int32_t)strlen(fraction);
+
+  if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
+    if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
+        tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+static bool validateTimezoneFormat(const SValueNode* pVal) {
+  if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
+    return false;
+  }
+
+  char*   tz = varDataVal(pVal->datum.p);
+  int32_t len = varDataLen(pVal->datum.p);
+
+  char   buf[3] = {0};
+  int8_t hour = -1, minute = -1;
+  if (len == 0) {
+    return false;
+  } else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) {
+    return true;
+  } else if ((tz[0] == '+' || tz[0] == '-')) {
+    switch (len) {
+      case 3:
+      case 5: {
+        for (int32_t i = 1; i < len; ++i) {
+          if (!isdigit(tz[i])) {
+            return false;
+          }
+
+          if (i == 2) {
+            memcpy(buf, &tz[i - 1], 2);
+            hour = taosStr2Int8(buf, NULL, 10);
+            if (!validateHourRange(hour)) {
+              return false;
+            }
+          } else if (i == 4) {
+            memcpy(buf, &tz[i - 1], 2);
+            minute = taosStr2Int8(buf, NULL, 10);
+            if (!validateMinuteRange(hour, minute, tz[0])) {
+              return false;
+            }
+          }
+        }
+        break;
+      }
+      case 6: {
+        for (int32_t i = 1; i < len; ++i) {
+          if (i == 3) {
+            if (tz[i] != ':') {
+              return false;
+            }
+            continue;
+          }
+
+          if (!isdigit(tz[i])) {
+            return false;
+          }
+
+          if (i == 2) {
+            memcpy(buf, &tz[i - 1], 2);
+            hour = taosStr2Int8(buf, NULL, 10);
+            if (!validateHourRange(hour)) {
+              return false;
+            }
+          } else if (i == 5) {
+            memcpy(buf, &tz[i - 1], 2);
+            minute = taosStr2Int8(buf, NULL, 10);
+            if (!validateMinuteRange(hour, minute, tz[0])) {
+              return false;
+            }
+          }
+        }
+        break;
+      }
+      default: {
+        return false;
+      }
+    }
+  } else {
+    return false;
+  }
+
+  return true;
+}
+
+void static addTimezoneParam(SNodeList* pList) {
+  char       buf[6] = {0};
+  time_t     t = taosTime(NULL);
+  struct tm* tmInfo = taosLocalTime(&t, NULL);
+  strftime(buf, sizeof(buf), "%z", tmInfo);
+  int32_t len = (int32_t)strlen(buf);
+
+  SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
+  pVal->literal = strndup(buf, len);
+  pVal->isDuration = false;
+  pVal->translate = true;
+  pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
+  pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
+  pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
+  pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1);
+  varDataSetLen(pVal->datum.p, len);
+  strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
+
+  nodesListAppend(pList, (SNode*)pVal);
+}
+
 void static addDbPrecisonParam(SNodeList** pList, uint8_t precision) {
   SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
   pVal->literal = NULL;
@@ -525,9 +693,15 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len
       return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
     }
 
-    if (pValue->datum.i == 0) {
+    uint8_t dbPrec = pFunc->node.resType.precision;
+
+    int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1));
+    if (ret == TIME_UNIT_TOO_SMALL) {
       return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                              "ELAPSED function time unit parameter should be greater than db precision");
+    } else if (ret == TIME_UNIT_INVALID) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "ELAPSED function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
     }
   }
 
@@ -843,6 +1017,19 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
+  if (numOfParams == 4) {
+    uint8_t dbPrec = pFunc->node.resType.precision;
+
+    int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 3));
+    if (ret == TIME_UNIT_TOO_SMALL) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "STATEDURATION function time unit parameter should be greater than db precision");
+    } else if (ret == TIME_UNIT_INVALID) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "STATEDURATION function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+    }
+  }
+
   // set result type
   pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
   return TSDB_CODE_SUCCESS;
@@ -1259,19 +1446,9 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
 
 static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // The number of parameters has been limited by the syntax definition
-  // uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
 
   // The function return type has been set during syntax parsing
   uint8_t para2Type = pFunc->node.resType.type;
-  // if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
-  //     para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
-  //     para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
-  //   return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
-  // }
-  // if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
-  //     (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
-  //   return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
-  // }
 
   int32_t para2Bytes = pFunc->node.resType.bytes;
   if (IS_VAR_DATA_TYPE(para2Type)) {
@@ -1281,153 +1458,12 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
     return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                            "CAST function converted length should be in range [0, 1000]");
   }
-  return TSDB_CODE_SUCCESS;
-}
-
-/* Following are valid ISO-8601 timezone format:
- * 1 z/Z
- * 2 ±hh:mm
- * 3 ±hhmm
- * 4 ±hh
- *
- */
-
-static bool validateHourRange(int8_t hour) {
-  if (hour < 0 || hour > 12) {
-    return false;
-  }
-
-  return true;
-}
-
-static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
-  if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '+')) {
-    return true;
-  }
-
-  return false;
-}
-
-static bool validateTimestampDigits(const SValueNode* pVal) {
-  if (!IS_INTEGER_TYPE(pVal->node.resType.type)) {
-    return false;
-  }
-
-  int64_t tsVal = pVal->datum.i;
-  char    fraction[20] = {0};
-  NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction);
-  int32_t tsDigits = (int32_t)strlen(fraction);
-
-  if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
-    if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
-        tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
-      return true;
-    } else {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-static bool validateTimezoneFormat(const SValueNode* pVal) {
-  if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
-    return false;
-  }
-
-  char*   tz = varDataVal(pVal->datum.p);
-  int32_t len = varDataLen(pVal->datum.p);
-
-  char   buf[3] = {0};
-  int8_t hour = -1, minute = -1;
-  if (len == 0) {
-    return false;
-  } else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) {
-    return true;
-  } else if ((tz[0] == '+' || tz[0] == '-')) {
-    switch (len) {
-      case 3:
-      case 5: {
-        for (int32_t i = 1; i < len; ++i) {
-          if (!isdigit(tz[i])) {
-            return false;
-          }
-
-          if (i == 2) {
-            memcpy(buf, &tz[i - 1], 2);
-            hour = taosStr2Int8(buf, NULL, 10);
-            if (!validateHourRange(hour)) {
-              return false;
-            }
-          } else if (i == 4) {
-            memcpy(buf, &tz[i - 1], 2);
-            minute = taosStr2Int8(buf, NULL, 10);
-            if (!validateMinuteRange(hour, minute, tz[0])) {
-              return false;
-            }
-          }
-        }
-        break;
-      }
-      case 6: {
-        for (int32_t i = 1; i < len; ++i) {
-          if (i == 3) {
-            if (tz[i] != ':') {
-              return false;
-            }
-            continue;
-          }
-
-          if (!isdigit(tz[i])) {
-            return false;
-          }
-
-          if (i == 2) {
-            memcpy(buf, &tz[i - 1], 2);
-            hour = taosStr2Int8(buf, NULL, 10);
-            if (!validateHourRange(hour)) {
-              return false;
-            }
-          } else if (i == 5) {
-            memcpy(buf, &tz[i - 1], 2);
-            minute = taosStr2Int8(buf, NULL, 10);
-            if (!validateMinuteRange(hour, minute, tz[0])) {
-              return false;
-            }
-          }
-        }
-        break;
-      }
-      default: {
-        return false;
-      }
-    }
-  } else {
-    return false;
-  }
 
-  return true;
-}
-
-void static addTimezoneParam(SNodeList* pList) {
-  char       buf[6] = {0};
-  time_t     t = taosTime(NULL);
-  struct tm* tmInfo = taosLocalTime(&t, NULL);
-  strftime(buf, sizeof(buf), "%z", tmInfo);
-  int32_t len = (int32_t)strlen(buf);
-
-  SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
-  pVal->literal = strndup(buf, len);
-  pVal->isDuration = false;
-  pVal->translate = true;
-  pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
-  pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
-  pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
-  pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1);
-  varDataSetLen(pVal->datum.p, len);
-  strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
+  // add database precision as param
+  uint8_t dbPrec = pFunc->node.resType.precision;
+  addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
 
-  nodesListAppend(pList, (SNode*)pVal);
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
@@ -1498,6 +1534,16 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
 
   // add database precision as param
   uint8_t dbPrec = pFunc->node.resType.precision;
+
+  int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1));
+  if (ret == TIME_UNIT_TOO_SMALL) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "TIMETRUNCATE function time unit parameter should be greater than db precision");
+  } else if (ret == TIME_UNIT_INVALID) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "TIMETRUNCATE function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+  }
+
   addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
 
   pFunc->node.resType =
@@ -1526,6 +1572,18 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
 
   // add database precision as param
   uint8_t dbPrec = pFunc->node.resType.precision;
+
+  if (3 == numOfParams) {
+    int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 2));
+    if (ret == TIME_UNIT_TOO_SMALL) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "TIMEDIFF function time unit parameter should be greater than db precision");
+    } else if (ret == TIME_UNIT_INVALID) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "TIMEDIFF function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+    }
+  }
+
   addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
 
   pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};

source/libs/index/CMakeLists.txt

@@ -35,7 +35,7 @@ if (${BUILD_WITH_INVERTEDINDEX})
 endif(${BUILD_WITH_INVERTEDINDEX})
 
 
-if (${BUILD_TEST})
-  add_subdirectory(test)
-endif(${BUILD_TEST})
+# if (${BUILD_TEST})
+#   add_subdirectory(test)
+# endif(${BUILD_TEST})
 

source/libs/scalar/src/sclfunc.c

@@ -934,9 +934,16 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
         break;
       }
       case TSDB_DATA_TYPE_TIMESTAMP: {
+        int64_t timeVal;
         if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) {
-          //convert to 0
+          int64_t timePrec;
+          GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
+          int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal);
+          if (ret != TSDB_CODE_SUCCESS) {
             *(int64_t *)output = 0;
+          } else {
+            *(int64_t *)output = timeVal;
+          }
         } else {
           GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input);
         }

source/libs/scheduler/inc/schedulerInt.h

@@ -276,8 +276,6 @@ extern SSchedulerMgmt schMgmt;
 
 #define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1)
 #define SCH_TASK_EID(_task) ((_task) ? (_task)->execId : -1)
-#define SCH_SET_TASK_LASTMSG_TYPE(_task, _type) do { if(_task) { atomic_store_32(&(_task)->lastMsgType, _type); } } while (0)
-#define SCH_GET_TASK_LASTMSG_TYPE(_task) ((_task) ? atomic_load_32(&(_task)->lastMsgType) : -1)
 
 #define SCH_IS_DATA_SRC_QRY_TASK(task) ((task)->plan->subplanType == SUBPLAN_TYPE_SCAN)
 #define SCH_IS_DATA_SRC_TASK(task) (((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) || ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY))
@@ -309,7 +307,10 @@ extern SSchedulerMgmt schMgmt;
 #define SCH_IS_NEED_DROP_JOB(_job) (SCH_IS_QUERY_JOB(_job))
 #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode)
 #define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
-#define SCH_SUB_TASK_NETWORK_ERR(_code, _len) (((_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_RPC_BROKEN_LINK) && ((_len) > 0))
+#define SCH_SUB_TASK_NETWORK_ERR(_code, _len) (SCH_NETWORK_ERR(_code) && ((_len) > 0))
+#define SCH_NEED_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH)
+#define SCH_NEED_REDIRECT(_msgType, _code, _rspLen) (SCH_NEED_REDIRECT_MSGTYPE(_msgType) && (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_SUB_TASK_NETWORK_ERR(_code, _rspLen)))
+#define SCH_NEED_RETRY(_msgType, _code) ((SCH_NETWORK_ERR(_code) && SCH_NEED_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
 
 #define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum)
 #define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse])

source/libs/scheduler/src/schJob.c

@@ -835,7 +835,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
     return TSDB_CODE_SUCCESS;
   }
 
-  if (!NEED_SCHEDULER_RETRY_ERROR(errCode)) {
+  if (!SCH_NEED_RETRY(pTask->lastMsgType, errCode)) {
     *needRetry = false;
     SCH_TASK_DLOG("task no more retry cause of errCode, errCode:%x - %s", errCode, tstrerror(errCode));
     return TSDB_CODE_SUCCESS;

source/libs/scheduler/src/schRemote.c

@@ -23,7 +23,7 @@
 
 
 int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) {
-  int32_t lastMsgType = SCH_GET_TASK_LASTMSG_TYPE(pTask);
+  int32_t lastMsgType = pTask->lastMsgType;
   int32_t taskStatus = SCH_GET_TASK_STATUS(pTask);
   int32_t reqMsgType = msgType - 1;
   switch (msgType) {
@@ -42,7 +42,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy
                       TMSG_INFO(msgType));
       }
 
-      SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
+      //SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
       return TSDB_CODE_SUCCESS;
     case TDMT_SCH_FETCH_RSP:
       if (lastMsgType != reqMsgType && -1 != lastMsgType) {
@@ -57,7 +57,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy
         SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
       }
 
-      SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
+      //SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
       return TSDB_CODE_SUCCESS;
     case TDMT_VND_CREATE_TABLE_RSP:
     case TDMT_VND_DROP_TABLE_RSP:
@@ -82,7 +82,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy
     SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
   }
 
-  SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
+  //SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -396,7 +396,8 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) {
 
   SCH_ERR_JRET(schValidateReceivedMsgType(pJob, pTask, msgType));
 
-  if (NEED_SCHEDULER_REDIRECT_ERROR(rspCode) || SCH_SUB_TASK_NETWORK_ERR(rspCode, pMsg->len > 0)) {
+  int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1);
+  if (SCH_NEED_REDIRECT(reqType, rspCode, pMsg->len)) {
     code = schHandleRedirect(pJob, pTask, (SDataBuf *)pMsg, rspCode);
     goto _return;
   }
@@ -855,6 +856,9 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery
          addr->nodeId, epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, 
          trans->pTrans, trans->pHandle);
 
+  if (pTask) {
+    pTask->lastMsgType = msgType;
+  }
 
   int64_t transporterId = 0;
   code = asyncSendMsgToServerExt(trans->pTrans, epSet, &transporterId, pMsgSendInfo, persistHandle, ctx);
@@ -1098,8 +1102,6 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
       break;
   }
 
-  SCH_SET_TASK_LASTMSG_TYPE(pTask, msgType);
-
   SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)};
   SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle,
                                (rpcCtx.args ? &rpcCtx : NULL)));
@@ -1112,7 +1114,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
 
 _return:
 
-  SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
+  pTask->lastMsgType = -1;
   schFreeRpcCtx(&rpcCtx);
 
   taosMemoryFreeClear(msg);

source/os/src/osString.c

@@ -196,7 +196,7 @@ int32_t taosUcs4len(TdUcs4 *ucs4) {
 }
 
 //dst buffer size should be at least 2*len + 1
-int32_t taosHexEncode(const char *src, char *dst, int32_t len) {
+int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len) {
   if (!dst) {
     return -1;
   }

source/util/src/terror.c

@@ -590,6 +590,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE,         "No tsma index in ca
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV,               "Invalid rsma env")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT,              "Invalid rsma state")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_QTASKINFO_CREATE,          "Rsma qtaskinfo creation error")
+TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FILE_CORRUPTED,            "Rsma file corrupted")
+
 
 //tq
 TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET,         "No committed offset")

tests/pytest/util/common.py

@@ -663,6 +663,12 @@ class TDCom:
         else:
             tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}")
             
+    def killProcessor(self, processorName):        
+        if (platform.system().lower() == 'windows'):
+            os.system("TASKKILL /F /IM %s.exe"%processorName)
+        else:
+            os.system('pkill %s'%processorName)        
+
 
 def is_json(msg):
     if isinstance(msg, str):

tests/script/jenkins/basic.txt

@@ -93,6 +93,7 @@
 ./test.sh -f tsim/stream/basic0.sim
 ./test.sh -f tsim/stream/basic1.sim
 ./test.sh -f tsim/stream/basic2.sim
+./test.sh -f tsim/stream/drop_stream.sim
 ./test.sh -f tsim/stream/distributeInterval0.sim
 # ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim
 # ./test.sh -f tsim/stream/distributesession0.sim
@@ -105,6 +106,7 @@
 ./test.sh -f tsim/stream/partitionby1.sim
 ./test.sh -f tsim/stream/schedSnode.sim
 ./test.sh -f tsim/stream/windowClose.sim
+./test.sh -f tsim/stream/ignoreExpiredData.sim
 
 # ---- transaction
  ./test.sh -f tsim/trans/lossdata1.sim
@@ -159,6 +161,7 @@
 #./test.sh -f tsim/mnode/basic1.sim -m
 
 # --- sma
+./test.sh -f tsim/sma/drop_sma.sim
 ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
 ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
 

tests/script/sh/deploy.sh

@@ -135,7 +135,7 @@ echo "jniDebugFlag           143"                >> $TAOS_CFG
 echo "qDebugFlag             143"                >> $TAOS_CFG
 echo "rpcDebugFlag           143"                >> $TAOS_CFG
 echo "tmrDebugFlag           131"                >> $TAOS_CFG
-echo "uDebugFlag             143"                >> $TAOS_CFG
+echo "uDebugFlag             131"                >> $TAOS_CFG
 echo "sDebugFlag             143"                >> $TAOS_CFG
 echo "wDebugFlag             143"                >> $TAOS_CFG
 echo "numOfLogLines          20000000"           >> $TAOS_CFG

tests/script/tsim/sma/drop_sma.sim

@@ -64,12 +64,59 @@ sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) in
 print --> drop stb
 sql drop table stb; 
 
+print ========== step5 
+sql drop database if exists db;
+sql create database db duration 300;
+sql use db;
+sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint, c_float float, c_double double, c_bool bool, c_binary binary(16), c_nchar nchar(32), c_ts timestamp, c_tint_un tinyint unsigned, c_sint_un smallint unsigned, c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);
+sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
+
+print ========== step6 repeat
+sql drop database if exists db;
+sql create database db duration 300;
+sql use db;
+sql create table stb1(ts timestamp, c_int int, c_bint bigint ) tags (t_int int);
+sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
+
+print ========== step7
+sql drop database if exists db;
+sql create database db duration 300;
+sql use db;
+sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);
+
+sql create table ct1 using stb1 tags ( 1 );
+sql create table ct2 using stb1 tags ( 2 );
+sql create table ct3 using stb1 tags ( 3 );
+sql create table ct4 using stb1 tags ( 4 );
+
+sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
+sql CREATE SMA INDEX sma_index_2 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) max_delay 6m;
+sql CREATE SMA INDEX sma_index_3 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) watermark 5s max_delay 6m;
+
+sql DROP INDEX sma_index_1 ;
+sql DROP INDEX sma_index_2 ;
+sql DROP INDEX sma_index_3 ;
+
+print ========== step8
+sql drop database if exists db;
+sql create database db duration 300;
+sql use db;
+sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);
+
+sql create table ct1 using stb1 tags ( 1 );
+sql create table ct2 using stb1 tags ( 2 );
+sql create table ct3 using stb1 tags ( 3 );
+sql create table ct4 using stb1 tags ( 4 );
+
+sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
+sql CREATE SMA INDEX sma_index_2 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) max_delay 6m;
+sql CREATE SMA INDEX sma_index_3 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) watermark 5s max_delay 6m;
+
+sql DROP INDEX sma_index_1 ;
+sql DROP INDEX sma_index_2 ;
+sql DROP INDEX sma_index_3 ;
 
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
 system sh/exec.sh -n dnode4 -s stop  -x SIGINT
\ No newline at end of file
-system sh/exec.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/tsim/stream/distributeInterval0.sim

@@ -83,9 +83,9 @@ sql insert into ts3 values(1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0)
 
 $loop_count = 0
 loop1:
+sleep 300
 sql select * from streamtST1;
 
-sleep 300
 $loop_count = $loop_count + 1
 if $loop_count == 10 then
   return -1

tests/script/tsim/stream/distributeSession0.sim

@@ -10,6 +10,30 @@ sql create dnode $hostname2 port 7200
 
 system sh/exec.sh -n dnode2 -s start
 
+print ===== step1
+$x = 0
+step1:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 2 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+
+print ===== step2
+
 sql create database test  vgroups 4;
 sql use test;
 sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);

tests/script/tsim/stream/ignoreExpiredData.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+
+system sh/exec.sh -n dnode1 -s start 
+sleep 50
+sql connect
+
+sql create dnode $hostname2 port 7200
+
+system sh/exec.sh -n dnode2 -s start
+
+print ===== step1
+$x = 0
+step1:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 2 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+
+print ===== step2
+
+print =============== create database
+sql create database test vgroups 1
+sql show databases
+if $rows != 3 then
+  return -1
+endi
+
+print $data00 $data01 $data02
+
+sql use test
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams1 trigger at_once IGNORE EXPIRED into streamt1 as select  _wstartts, count(*) c1, sum(a) c3 from t1 interval(10s);
+sql create stream streams2 trigger at_once IGNORE EXPIRED into streamt2 as select  _wstartts, count(*) c1, sum(a) c3 from t1 session(ts,10s);
+sql create stream streams3 trigger at_once IGNORE EXPIRED into streamt3 as select  _wstartts, count(*) c1, sum(a) c3 from t1 state_window(a);
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.1);
+sql insert into t1 values(1648791233002,2,2,3,2.1);
+sql insert into t1 values(1648791243003,2,2,3,3.1);
+sql insert into t1 values(1648791200000,4,2,3,4.1);
+
+$loop_count = 0
+loop1:
+sleep 300
+sql select * from streamt1;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 4 then
+  print =====rows=$rows
+  goto loop1
+endi
+
+$loop_count = 0
+loop2:
+sleep 300
+sql select * from streamt2;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 4 then
+  print =====rows=$rows
+  goto loop2
+endi
+
+$loop_count = 0
+loop3:
+sleep 300
+sql select * from streamt3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 2 then
+  print =====rows=$rows
+  goto loop3
+endi
+
+
+print =============== create database
+sql create database test1 vgroups 4
+sql show databases
+
+print ======database=$rows
+
+sql use test1
+
+sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create table ts2 using st tags(2,2,2);
+sql create stream stream_t1 trigger at_once IGNORE EXPIRED into streamtST1 as select  _wstartts, count(*) c1, count(a) c2 , sum(a) c3 ,  max(b)  c5, min(c) c6 from st interval(10s) ;
+sql create stream stream_t2 trigger at_once IGNORE EXPIRED into streamtST2 as select  _wstartts, count(*) c1, count(a) c2 , sum(a) c3 ,  max(b)  c5, min(c) c6 from st session(ts, 10s) ;
+sql insert into ts1 values(1648791211000,1,2,3);
+sql insert into ts1 values(1648791222001,2,2,3);
+sql insert into ts2 values(1648791211000,1,2,3);
+sql insert into ts2 values(1648791222001,2,2,3);
+
+$loop_count = 0
+loop4:
+sleep 300
+sql select * from streamtST1;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop4
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop4
+endi
+
+$loop_count = 0
+loop5:
+sleep 300
+sql select * from streamtST2;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop5
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop5
+endi
+
+system sh/stop_dnodes.sh
\ No newline at end of file

tests/script/tsim/tmq/snapshot1.sim

@@ -25,7 +25,7 @@ $rowsPerCtb = 10
 $tstart     = 1640966400000  # 2022-01-01 00:00:00.000
 #---- global parameters end ----#
 
-$pullDelay    = 5
+$pullDelay    = 2
 $ifcheckdata  = 1
 $ifmanualcommit = 1
 $showMsg      = 1

tests/system-test/1-insert/insertWithMoreVgroup.py

@@ -355,40 +355,6 @@ class TDTestCase:
 
         return 
 
-    def test_case4(self):
-        self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 1, 2, 1*10)
-        tdSql.execute("use db1;")
-        tdSql.query("show dnodes;")
-        dnodeId=tdSql.getData(0,0)
-        print(dnodeId)
-        tdSql.execute("create qnode on dnode %s"%dnodeId)
-        tdSql.query("select max(c1) from stb10;")
-        maxQnode=tdSql.getData(0,0)
-        tdSql.query("select min(c1) from stb11;")
-        minQnode=tdSql.getData(0,0)
-        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;")
-        unionQnode=tdSql.queryResult
-        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all  select c0,c1 from stb11_1 where c0>2000;")
-        unionallQnode=tdSql.queryResult
-
-        # tdSql.query("show qnodes;")
-        # qnodeId=tdSql.getData(0,0)
-        tdSql.execute("drop qnode on dnode %s"%dnodeId)
-        tdSql.execute("reset query cache")
-        tdSql.query("select max(c1) from stb10;")
-        tdSql.checkData(0, 0, "%s"%maxQnode)
-        tdSql.query("select min(c1) from stb11;")     
-        tdSql.checkData(0, 0, "%s"%minQnode)
-        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;")
-        unionVnode=tdSql.queryResult
-        assert unionQnode == unionVnode
-        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all  select c0,c1 from stb11_1 where c0>2000;")
-        unionallVnode=tdSql.queryResult
-        assert unionallQnode == unionallVnode
-
-
-        # tdSql.execute("create qnode on dnode %s"%dnodeId)
-
     # run case   
     def run(self):
 

tests/system-test/1-insert/test_stmt_insert_query_ex.py

@@ -82,7 +82,7 @@ class TDTestCase:
         return con
         
     def test_stmt_set_tbname_tag(self,conn):
-        dbname = "stmt_set_tbname_tag"
+        dbname = "stmt_tag"
         
         try:
             conn.execute("drop database if exists %s" % dbname)
@@ -196,31 +196,31 @@ class TDTestCase:
             assert rows9[0][0] == 12, 'fourth case is failed'
             assert rows9[1][0] == 12, 'fourth case is failed'
 
-            # #query: conversion Functions
-
-            # querystmt4=conn.statement("select cast( ? as bigint) from log  ")
-            # queryparam4=new_bind_params(1)
-            # print(type(queryparam4))
-            # queryparam4[0].binary('1232a')
-            # querystmt4.bind_param(queryparam4)
-            # querystmt4.execute() 
-            # result4=querystmt4.use_result()
-            # rows4=result4.fetch_all()
-            # print("5",rows4)
-            # assert rows4[0][0] == 1232
-            # assert rows4[1][0] == 1232
-
-            # querystmt4=conn.statement("select cast( ? as binary(10)) from log  ")
-            # queryparam4=new_bind_params(1)
-            # print(type(queryparam4))
-            # queryparam4[0].int(123)
-            # querystmt4.bind_param(queryparam4)
-            # querystmt4.execute() 
-            # result4=querystmt4.use_result()
-            # rows4=result4.fetch_all()
-            # print("6",rows4)
-            # assert rows4[0][0] == '123'
-            # assert rows4[1][0] == '123'
+            #query: conversion Functions
+
+            querystmt4=conn.statement("select cast( ? as bigint) from log  ")
+            queryparam4=new_bind_params(1)
+            print(type(queryparam4))
+            queryparam4[0].binary('1232a')
+            querystmt4.bind_param(queryparam4)
+            querystmt4.execute() 
+            result4=querystmt4.use_result()
+            rows4=result4.fetch_all()
+            print("5",rows4)
+            assert rows4[0][0] == 1232
+            assert rows4[1][0] == 1232
+
+            querystmt4=conn.statement("select cast( ? as binary(10)) from log  ")
+            queryparam4=new_bind_params(1)
+            print(type(queryparam4))
+            queryparam4[0].int(123)
+            querystmt4.bind_param(queryparam4)
+            querystmt4.execute() 
+            result4=querystmt4.use_result()
+            rows4=result4.fetch_all()
+            print("6",rows4)
+            assert rows4[0][0] == '123'
+            assert rows4[1][0] == '123'
 
             # #query: datatime Functions
 

tests/system-test/1-insert/test_stmt_muti_insert_query.py

@@ -84,21 +84,21 @@ class TDTestCase:
     def test_stmt_insert_multi(self,conn):
         # type: (TaosConnection) -> None
 
-        dbname = "pytest_taos_stmt_multi"
+        dbname = "db_stmt"
         try:
             conn.execute("drop database if exists %s" % dbname)
             conn.execute("create database if not exists %s" % dbname)
             conn.select_db(dbname)
 
             conn.execute(
-                "create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\
+                "create table if not exists stb1(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\
                 bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \
                 ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)",
             )
             # conn.load_table_info("log")
 
             start = datetime.now()
-            stmt = conn.statement("insert into log values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
+            stmt = conn.statement("insert into stb1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
 
             params = new_multi_binds(16)
             params[0].timestamp((1626861392589, 1626861392590, 1626861392591))
@@ -125,7 +125,7 @@ class TDTestCase:
             assert stmt.affected_rows == 3
             
             #query 1
-            querystmt=conn.statement("select ?,bu from log")
+            querystmt=conn.statement("select ?,bu from stb1")
             queryparam=new_bind_params(1)
             print(type(queryparam))
             queryparam[0].binary("ts")
@@ -135,7 +135,7 @@ class TDTestCase:
             # rows=result.fetch_all()
             # print( querystmt.use_result())
 
-            # result = conn.query("select * from log")
+            # result = conn.query("select * from stb1")
             rows=result.fetch_all()
             # rows=result.fetch_all()
             print(rows)
@@ -144,7 +144,7 @@ class TDTestCase:
             assert rows[2][1] == None
 
             #query 2
-            querystmt1=conn.statement("select * from log where bu < ?")
+            querystmt1=conn.statement("select * from stb1 where bu < ?")
             queryparam1=new_bind_params(1)
             print(type(queryparam1))
             queryparam1[0].int(4)

tests/system-test/1-insert/test_stmt_set_tbname_tag.py

@@ -78,21 +78,29 @@ class TDTestCase:
         password = "taosdata"
         port =6030 
         con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port)
-        print(con)
+        tdLog.debug(con)
         return con
 
-    def test_stmt_set_tbname_tag(self,conn):
-        dbname = "stmt_set_tbname_tag"
+    def stmtExe(self,conn,sql,bindStat):
+            queryStat=conn.statement("%s"%sql)
+            queryStat.bind_param(bindStat)
+            queryStat.execute() 
+            result=queryStat.use_result()
+            rows=result.fetch_all()
+            return rows
 
+    def test_stmt_set_tbname_tag(self,conn):
+        dbname = "stmt_tag"
+        stablename = 'log'
         try:
             conn.execute("drop database if exists %s" % dbname)
             conn.execute("create database if not exists %s PRECISION 'us' "  % dbname)
             conn.select_db(dbname)
-            conn.execute("create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\
+            conn.execute("create table if not exists %s(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\
                 bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \
                 ff float, dd double, bb binary(100), nn nchar(100), tt timestamp , vc varchar(100)) tags (t1 timestamp, t2 bool,\
                 t3 tinyint, t4 tinyint, t5 smallint, t6 int, t7 bigint, t8 tinyint unsigned, t9 smallint unsigned, \
-                t10 int unsigned, t11 bigint unsigned, t12 float, t13 double, t14 binary(100), t15 nchar(100), t16 timestamp)")
+                t10 int unsigned, t11 bigint unsigned, t12 float, t13 double, t14 binary(100), t15 nchar(100), t16 timestamp)"%stablename)
             
             stmt = conn.statement("insert into ? using log tags (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) \
                 values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
@@ -139,142 +147,93 @@ class TDTestCase:
             assert stmt.affected_rows == 3
 
             #query all 
-            querystmt1=conn.statement("select * from log where bu < ?")
-            queryparam1=new_bind_params(1)
-            print(type(queryparam1))
-            queryparam1[0].int(10)
-            querystmt1.bind_param(queryparam1)
-            querystmt1.execute() 
-            result1=querystmt1.use_result()
-            rows1=result1.fetch_all()
-            print(rows1[0])
-            print(rows1[1])
-            print(rows1[2])
-            assert str(rows1[0][0]) == "2021-07-21 17:56:32.589111"
-            assert rows1[0][10] == 3
-            assert rows1[1][10] == 4
+            queryparam=new_bind_params(1)
+            queryparam[0].int(10)
+            rows=self.stmtExe(conn,"select * from log where bu < ?",queryparam)
+            tdLog.debug("assert 1st case %s"%rows)
+            assert str(rows[0][0]) == "2021-07-21 17:56:32.589111"
+            assert rows[0][10] == 3 , '1st case is failed'
+            assert rows[1][10] == 4 , '1st case is failed'
 
             #query: Numeric Functions
-            querystmt2=conn.statement("select abs(?) from log where bu < ?")
-            queryparam2=new_bind_params(2)
-            print(type(queryparam2))
-            queryparam2[0].int(5)
-            queryparam2[1].int(5)
-            querystmt2.bind_param(queryparam2)
-            querystmt2.execute() 
-            result2=querystmt2.use_result()
-            rows2=result2.fetch_all()
-            print("2",rows2)
-            assert rows2[0][0] == 5
-            assert rows2[1][0] == 5
+            queryparam=new_bind_params(2)
+            queryparam[0].int(5)
+            queryparam[1].int(5)
+            rows=self.stmtExe(conn,"select abs(?) from log where bu < ?",queryparam)
+            tdLog.debug("assert 2nd case %s"%rows)
+            assert rows[0][0] == 5 , '2nd case is failed'
+            assert rows[1][0] == 5 , '2nd case is failed'
 
 
             #query: Numeric Functions and escapes
-
-            querystmt3=conn.statement("select abs(?) from log where  nn= 'a? long string with 中文字符' ")
-            queryparam3=new_bind_params(1)
-            print(type(queryparam3))
-            queryparam3[0].int(5)
-            querystmt3.bind_param(queryparam3)
-            querystmt3.execute() 
-            result3=querystmt3.use_result()
-            rows3=result3.fetch_all()
-            print("3",rows3)
-            assert rows3 == []
-
-            # #query: string Functions
-
-            # querystmt3=conn.statement("select CHAR_LENGTH(?) from log  ")
-            # queryparam3=new_bind_params(1)
-            # print(type(queryparam3))
-            # queryparam3[0].binary('中文字符')
-            # querystmt3.bind_param(queryparam3)
-            # querystmt3.execute() 
-            # result3=querystmt3.use_result()
-            # rows3=result3.fetch_all()
-            # print("4",rows3)
-            # assert rows3[0][0] == 12, 'fourth case is failed'
-            # assert rows3[1][0] == 12, 'fourth case is failed'
-
-            # #query: conversion Functions
-
-            # querystmt4=conn.statement("select cast( ? as bigint) from log  ")
-            # queryparam4=new_bind_params(1)
-            # print(type(queryparam4))
-            # queryparam4[0].binary('1232a')
-            # querystmt4.bind_param(queryparam4)
-            # querystmt4.execute() 
-            # result4=querystmt4.use_result()
-            # rows4=result4.fetch_all()
-            # print("5",rows4)
-            # assert rows4[0][0] == 1232
-            # assert rows4[1][0] == 1232
-
-            # querystmt4=conn.statement("select cast( ? as binary(10)) from log  ")
-            # queryparam4=new_bind_params(1)
-            # print(type(queryparam4))
-            # queryparam4[0].int(123)
-            # querystmt4.bind_param(queryparam4)
-            # querystmt4.execute() 
-            # result4=querystmt4.use_result()
-            # rows4=result4.fetch_all()
-            # print("6",rows4)
-            # assert rows4[0][0] == '123'
-            # assert rows4[1][0] == '123'
-
-            # #query: datatime Functions
-
-            # querystmt4=conn.statement(" select timediff('2021-07-21 17:56:32.590111',?,1s)  from log  ")
-            # queryparam4=new_bind_params(1)
-            # print(type(queryparam4))
-            # queryparam4[0].timestamp(1626861392591111)
-            # querystmt4.bind_param(queryparam4)
-            # querystmt4.execute() 
-            # result4=querystmt4.use_result()
-            # rows4=result4.fetch_all()
-            # print("7",rows4)
-            # assert rows4[0][0] == 1, 'seventh case is failed'
-            # assert rows4[1][0] == 1, 'seventh case is failed'
+            queryparam=new_bind_params(1)
+            queryparam[0].int(5)
+            rows=self.stmtExe(conn,"select abs(?) from log where  nn= 'a? long string with 中文字符'",queryparam)
+            tdLog.debug("assert 3rd case %s"%rows)
+            assert rows == [] , '3rd case is failed'
+
+            #query: string Functions
+            queryparam=new_bind_params(1)
+            queryparam[0].binary('中文字符')
+            rows=self.stmtExe(conn,"select CHAR_LENGTH(?) from log ",queryparam)
+            tdLog.debug("assert 4th case %s"%rows)
+            assert rows[0][0] == 12, '4th case is failed'
+            assert rows[1][0] == 12, '4th case is failed'
+
+            queryparam=new_bind_params(1)
+            queryparam[0].binary('123')
+            rows=self.stmtExe(conn,"select CHAR_LENGTH(?) from log ",queryparam)
+            tdLog.debug("assert 4th case %s"%rows)
+            assert rows[0][0] == 3, '4th.1 case is failed'
+            assert rows[1][0] == 3, '4th.1 case is failed'
+
+            #query: conversion Functions
+            queryparam=new_bind_params(1)
+            queryparam[0].binary('1232a')           
+            rows=self.stmtExe(conn,"select cast( ? as bigint) from log",queryparam)
+            tdLog.debug("assert 5th case %s"%rows)
+            assert rows[0][0] == 1232, '5th.1 case is failed'
+            assert rows[1][0] == 1232, '5th.1 case is failed'
+
+            querystmt4=conn.statement("select cast( ? as binary(10)) from log  ")
+            queryparam=new_bind_params(1)
+            queryparam[0].int(123)
+            rows=self.stmtExe(conn,"select cast( ? as bigint) from log",queryparam)
+            tdLog.debug("assert 6th case %s"%rows)
+            assert rows[0][0] == 123, '6th.1 case is failed'
+            assert rows[1][0] == 123, '6th.1 case is failed'
+
+            #query: datatime Functions
+            queryparam=new_bind_params(1)
+            queryparam[0].timestamp(1626861392591112)
+            rows=self.stmtExe(conn,"select timediff('2021-07-21 17:56:32.590111',?,1a)  from log",queryparam)
+            tdLog.debug("assert 7th case %s"%rows)
+            assert rows[0][0] == 1, '7th case is failed'
+            assert rows[1][0] == 1, '7th case is failed'
             
             #query: aggregate Functions
-
-            querystmt4=conn.statement(" select count(?)  from log  ")
-            queryparam4=new_bind_params(1)
-            print(type(queryparam4))
-            queryparam4[0].int(123)
-            querystmt4.bind_param(queryparam4)
-            querystmt4.execute() 
-            result4=querystmt4.use_result()
-            rows4=result4.fetch_all()
-            print("8",rows4)
-            assert rows4[0][0] == 3, ' 8 case is failed'
-
-            #query: selector Functions 9
-
-            querystmt4=conn.statement("  select bottom(bu,?)  from log group by bu ;  ")
-            queryparam4=new_bind_params(1)
-            print(type(queryparam4))
-            queryparam4[0].int(2)
-            querystmt4.bind_param(queryparam4)
-            querystmt4.execute() 
-            result4=querystmt4.use_result()
-            rows4=result4.fetch_all()
-            print("9",rows4)
-            assert rows4[0][0] == 4, ' 9 case is failed'
-            assert rows4[1][0] == 3, ' 9 case is failed'
+            queryparam=new_bind_params(1)
+            queryparam[0].int(123)
+            rows=self.stmtExe(conn,"select count(?)  from log ",queryparam)
+            tdLog.debug("assert 8th case %s"%rows)
+            assert rows[0][0] == 3, ' 8th case is failed'
+
+            # #query: selector Functions 9
+            # queryparam=new_bind_params(1)
+            # queryparam[0].int(2)
+            # rows=self.stmtExe(conn,"select bottom(bu,?)  from log group by bu ; ",queryparam)
+            # tdLog.debug("assert 9th case %s"%rows)
+            # assert rows[0][0] == 4, ' 9 case is failed'
+            # assert rows[1][0] == 3, ' 9 case is failed'
 
             # #query: time-series specific Functions 10
 
-            querystmt4=conn.statement("  select twa(?)  from log;  ")
-            queryparam4=new_bind_params(1)
-            print(type(queryparam4))
-            queryparam4[0].int(15)
-            querystmt4.bind_param(queryparam4)
-            querystmt4.execute() 
-            result4=querystmt4.use_result()
-            rows4=result4.fetch_all()
-            print("10",rows4)
-            assert rows4[0][0] == 15, ' 10 case is failed'
+            querystmt=conn.statement("  select twa(?)  from log;  ")
+            queryparam=new_bind_params(1)
+            queryparam[0].int(15)
+            rows=self.stmtExe(conn," select twa(?)  from log; ",queryparam)
+            tdLog.debug("assert 10th case %s"%rows)
+            assert rows[0][0] == 15, ' 10th case is failed'
 
 
             # conn.execute("drop database if exists %s" % dbname)

tests/system-test/2-query/Timediff.py

@@ -72,12 +72,12 @@ class TDTestCase:
         tdSql.query("select timediff(1,0,1a) from db.ntb")
         tdSql.checkRows(3)
         tdSql.checkData(0,0,1000)
-        tdSql.query("select timediff(1,0,1u) from ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,1000000)
-        tdSql.query("select timediff(1,0,1u) from db.ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,1000000)
+        tdSql.error("select timediff(1,0,1u) from ntb")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,1000000)
+        tdSql.error("select timediff(1,0,1u) from db.ntb")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,1000000)
 
         tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00') from stb")
         tdSql.checkRows(3)
@@ -116,12 +116,12 @@ class TDTestCase:
         tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1a) from db.stb")
         tdSql.checkRows(3)
         tdSql.checkData(0,0,86400000)
-        tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,86400000000)
-        tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from db.stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,86400000000)
+        tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from stb")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,86400000000)
+        tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from db.stb")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,86400000000)
 
 
         tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00') from stb_1")
@@ -164,12 +164,12 @@ class TDTestCase:
         tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1a) from db.stb_1")
         tdSql.checkRows(3)
         tdSql.checkData(0,0,43200000)
-        tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from stb_1")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,43200000000)
-        tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from db.stb_1")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,43200000000)
+        tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from stb_1")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,43200000000)
+        tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from db.stb_1")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,43200000000)
 
         tdSql.query("select timediff('a','b') from stb")
         tdSql.checkRows(3)

tests/system-test/2-query/function_stateduration.py

@@ -348,10 +348,9 @@ class TDTestCase:
         tdSql.execute(" use db ")
         tdSql.error("select stateduration(c1,'GT',1,1b) from ct1")
         tdSql.error("select stateduration(c1,'GT',1,1u) from ct1")
+        tdSql.error("select stateduration(c1,'GT',1,1000s) from t1")
         tdSql.query("select stateduration(c1,'GT',1,1s) from t1")
         tdSql.checkData(10,0,63072035)
-        tdSql.query("select stateduration(c1,'GT',1,1000s) from t1")
-        tdSql.checkData(10,0,int(63072035/1000))
         tdSql.query("select stateduration(c1,'GT',1,1m) from t1")
         tdSql.checkData(10,0,int(63072035/60))
         tdSql.query("select stateduration(c1,'GT',1,1h) from t1")

tests/system-test/2-query/json_tag.py

@@ -675,7 +675,7 @@ class TDTestCase:
         tdSql.checkRows(3)
         tdSql.query("select TO_UNIXTIMESTAMP(datastr) from jsons1 where jtag->'tag1'>1;")
         tdSql.checkRows(3)
-        tdSql.query("select TIMETRUNCATE(ts,1u) from jsons1 where jtag->'tag1'>1;")
+        tdSql.query("select TIMETRUNCATE(ts,1s) from jsons1 where jtag->'tag1'>1;")
         tdSql.checkRows(3)
         tdSql.query("select TIMEDIFF(ts,_c0) from jsons1 where jtag->'tag1'>1;")
         tdSql.checkRows(3)

tests/system-test/2-query/queryQnode.py

@@ -17,6 +17,8 @@ import threading as thd
 import multiprocessing as mp
 from numpy.lib.function_base import insert
 import taos
+from util.dnodes import TDDnode
+from util.dnodes import *
 from util.log import *
 from util.cases import *
 from util.sql import *
@@ -30,9 +32,9 @@ class TDTestCase:
     #
     # --------------- main frame -------------------
     #
-    clientCfgDict = {'queryproxy': '1','debugFlag': 135}
-    clientCfgDict["queryproxy"] = '2'
-    clientCfgDict["debugFlag"] = 143
+    clientCfgDict = {'queryPolicy': '1','debugFlag': 135}
+    clientCfgDict["queryPolicy"] = '1'
+    clientCfgDict["debugFlag"] = 131
 
     updatecfgDict = {'clientCfg': {}}
     updatecfgDict = {'debugFlag': 143}
@@ -62,7 +64,7 @@ class TDTestCase:
         return buildPath
 
     # init
-    def init(self, conn, logSql):
+    def init(self, conn, logSql=True):
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor())
         # tdSql.prepare()
@@ -292,12 +294,13 @@ class TDTestCase:
         
         tdLog.debug("-----create database and muti-thread create tables test------- ")
 
-    def test_case4(self):
+    def test_case1(self):
         self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 1, 2, 1*10)
         tdSql.execute("use db1;")
         tdSql.query("show dnodes;")
         dnodeId=tdSql.getData(0,0)
         print(dnodeId)
+        tdLog.debug("create qnode on dnode %s"%dnodeId)
         tdSql.execute("create qnode on dnode %s"%dnodeId)
         tdSql.query("select max(c1) from stb10;")
         maxQnode=tdSql.getData(0,0)
@@ -310,6 +313,7 @@ class TDTestCase:
 
         # tdSql.query("show qnodes;")
         # qnodeId=tdSql.getData(0,0)
+        tdLog.debug("drop qnode on dnode %s"%dnodeId)
         tdSql.execute("drop qnode on dnode %s"%dnodeId)
         tdSql.execute("reset query cache")
         tdSql.query("select max(c1) from stb10;")
@@ -323,15 +327,156 @@ class TDTestCase:
         unionallVnode=tdSql.queryResult
         assert unionallQnode == unionallVnode
 
+        queryPolicy=2
+        simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath()
+        cmd='sed -i "s/^queryPolicy.*/queryPolicy 2/g" %s'%simClientCfg
+        os.system(cmd)
+        # tdDnodes.stop(1)
+        # tdDnodes.start(1)
+        tdSql.execute("reset query cache")
+        tdSql.execute('alter local  "queryPolicy" "%d"'%queryPolicy)
+        tdSql.query("show local variables;")
+        for i in range(tdSql.queryRows):
+            if tdSql.queryResult[i][0] == "queryPolicy" :
+                if int(tdSql.queryResult[i][1]) == int(queryPolicy):
+                    tdLog.success('alter queryPolicy to %d successfully'%queryPolicy)
+                else :
+                    tdLog.debug(tdSql.queryResult)
+                    tdLog.exit("alter queryPolicy to  %d failed"%queryPolicy)
+        tdSql.execute("reset query cache")
+
+        tdSql.execute("use db1;")
+        tdSql.query("show dnodes;")
+        dnodeId=tdSql.getData(0,0)
+        tdLog.debug("create qnode on dnode %s"%dnodeId)
+
+        tdSql.execute("create qnode on dnode %s"%dnodeId)
+        tdSql.query("select max(c1) from stb10;")
+        assert maxQnode==tdSql.getData(0,0)
+        tdSql.query("select min(c1) from stb11;")
+        assert minQnode==tdSql.getData(0,0)
+        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;")
+        assert unionQnode==tdSql.queryResult
+        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all  select c0,c1 from stb11_1 where c0>2000;")
+        assert unionallQnode==tdSql.queryResult
+
+        # tdSql.query("show qnodes;")
+        # qnodeId=tdSql.getData(0,0)
+        tdLog.debug("drop qnode on dnode %s"%dnodeId)
+        tdSql.execute("drop qnode on dnode %s"%dnodeId)
+        tdSql.execute("reset query cache")
+        tdSql.query("select max(c1) from stb10;")
+        assert maxQnode==tdSql.getData(0,0)
+        tdSql.query("select min(c1) from stb11;")  
+        assert minQnode==tdSql.getData(0,0)   
+        tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;")
+        tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union all  select c0,c1 from stb11_1 where c0>2000;")
 
         # tdSql.execute("create qnode on dnode %s"%dnodeId)
 
+        queryPolicy=3
+        simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath()
+        cmd='sed -i "s/^queryPolicy.*/queryPolicy 2/g" %s'%simClientCfg
+        os.system(cmd)
+        # tdDnodes.stop(1)
+        # tdDnodes.start(1)
+        tdSql.execute("reset query cache")
+        tdSql.execute('alter local  "queryPolicy" "%d"'%queryPolicy)
+        tdSql.query("show local variables;")
+        for i in range(tdSql.queryRows):
+            if tdSql.queryResult[i][0] == "queryPolicy" :
+                if int(tdSql.queryResult[i][1]) == int(queryPolicy):
+                    tdLog.success('alter queryPolicy to %d successfully'%queryPolicy)
+                else :
+                    tdLog.debug(tdSql.queryResult)
+                    tdLog.exit("alter queryPolicy to  %d failed"%queryPolicy)
+        tdSql.execute("reset query cache")
+
+        tdSql.execute("use db1;")
+        tdSql.query("show dnodes;")
+        dnodeId=tdSql.getData(0,0)
+        tdLog.debug("create qnode on dnode %s"%dnodeId)
+
+        tdSql.execute("create qnode on dnode %s"%dnodeId)
+        tdSql.query("select max(c1) from stb10;")
+        assert maxQnode==tdSql.getData(0,0)
+        tdSql.query("select min(c1) from stb11;")
+        assert minQnode==tdSql.getData(0,0)
+        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;")
+        assert unionQnode==tdSql.queryResult
+        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all  select c0,c1 from stb11_1 where c0>2000;")
+        assert unionallQnode==tdSql.queryResult
+
+    def test_case2(self):
+        self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 10, 2, 1*10)
+        tdSql.query("show qnodes")
+        if tdSql.queryRows == 1 :
+            tdLog.debug("drop qnode on dnode 1")
+            tdSql.execute("drop qnode on dnode 1")
+        queryPolicy=2
+        simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath()
+        cmd='sed -i "s/^queryPolicy.*/queryPolicy 2/g" %s'%simClientCfg
+        os.system(cmd)
+        # tdDnodes.stop(1)
+        # tdDnodes.start(1)
+        tdSql.execute("reset query cache")
+        tdSql.execute('alter local  "queryPolicy" "%d"'%queryPolicy)
+        tdSql.query("show local variables;")
+        for i in range(tdSql.queryRows):
+            if tdSql.queryResult[i][0] == "queryPolicy" :
+                if int(tdSql.queryResult[i][1]) == int(queryPolicy):
+                    tdLog.success('alter queryPolicy to %d successfully'%queryPolicy)
+                else :
+                    tdLog.debug(tdSql.queryResult)
+                    tdLog.exit("alter queryPolicy to  %d failed"%queryPolicy)
+        tdSql.execute("use db1;")
+        tdSql.error("select max(c1) from stb10;")
+        tdSql.error("select min(c1) from stb11;")     
+        tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;")
+        tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union all  select c0,c1 from stb11_1 where c0>2000;")
+
+        tdSql.query("select max(c1) from stb10_0;")
+        tdSql.query("select min(c1) from stb11_0;")
+
+    def test_case3(self):
+
+        tdSql.execute('alter local  "queryPolicy" "3"')
+        tdLog.debug("create qnode on dnode 1")
+        tdSql.execute("create qnode on dnode 1")
+        tdSql.execute("use db1;")
+        tdSql.query("show dnodes;")
+        dnodeId=tdSql.getData(0,0)
+        print(dnodeId)
+
+        tdSql.query("select max(c1) from stb10;")
+        maxQnode=tdSql.getData(0,0)
+        tdSql.query("select min(c1) from stb11;")
+        minQnode=tdSql.getData(0,0)
+        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;")
+        unionQnode=tdSql.queryResult
+        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all  select c0,c1 from stb11_1 where c0>2000;")
+        unionallQnode=tdSql.queryResult
+
+        # tdSql.query("show qnodes;")
+        # qnodeId=tdSql.getData(0,0)
+        tdLog.debug("drop qnode on dnode %s"%dnodeId)
+
+        tdSql.execute("drop qnode on dnode %s"%dnodeId)
+        tdSql.execute("reset query cache")
+
+        tdSql.error("select max(c1) from stb10;")
+        tdSql.error("select min(c1) from stb11;")     
+        tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;")
+        tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union all  select c0,c1 from stb11_1 where c0>2000;")
+
     # run case   
     def run(self):
-
         # test qnode
-        self.test_case4()
-        tdLog.debug(" LIMIT test_case3 ............ [OK]")
+        self.test_case1()
+        self.test_case2()
+
+        self.test_case3()
+        # tdLog.debug(" LIMIT test_case3 ............ [OK]")
 
 
         return 

tests/system-test/2-query/statecount.py

@@ -343,10 +343,9 @@ class TDTestCase:
         tdSql.execute(" use db ")
         tdSql.error("select stateduration(c1,'GT',1,1b) from ct1")
         tdSql.error("select stateduration(c1,'GT',1,1u) from ct1")
+        tdSql.error("select stateduration(c1,'GT',1,1000s) from t1")
         tdSql.query("select stateduration(c1,'GT',1,1s) from t1")
         tdSql.checkData(10,0,63072035)
-        tdSql.query("select stateduration(c1,'GT',1,1000s) from t1")
-        tdSql.checkData(10,0,int(63072035/1000))
         tdSql.query("select stateduration(c1,'GT',1,1m) from t1")
         tdSql.checkData(10,0,int(63072035/60))
         tdSql.query("select stateduration(c1,'GT',1,1h) from t1")

tests/system-test/2-query/timetruncate.py

@@ -37,8 +37,8 @@ class TDTestCase:
 
         tdSql.query("select timetruncate(1,1d) from ntb")
         tdSql.checkRows(10)
-        tdSql.query("select timetruncate(1,1u) from ntb")
-        tdSql.checkRows(10)
+        tdSql.error("select timetruncate(1,1u) from ntb")
+        #tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1a) from ntb")
         tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1m) from ntb")
@@ -97,8 +97,8 @@ class TDTestCase:
 
         tdSql.query("select timetruncate(1,1d) from stb")
         tdSql.checkRows(10)
-        tdSql.query("select timetruncate(1,1u) from stb")
-        tdSql.checkRows(10)
+        tdSql.error("select timetruncate(1,1u) from stb")
+        #tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1a) from stb")
         tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1m) from stb")
@@ -156,8 +156,8 @@ class TDTestCase:
 
         tdSql.query("select timetruncate(1,1d) from stb_1")
         tdSql.checkRows(10)
-        tdSql.query("select timetruncate(1,1u) from stb_1")
-        tdSql.checkRows(10)
+        tdSql.error("select timetruncate(1,1u) from stb_1")
+        #tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1a) from stb_1")
         tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1m) from stb_1")

tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py

@@ -65,31 +65,6 @@ class TDTestCase:
         self._async_raise(thread.ident, SystemExit)
 
 
-    def insertData(self,countstart,countstop):
-        # fisrt add data : db\stable\childtable\general table
-        
-        for couti in range(countstart,countstop):
-            tdLog.debug("drop database if exists db%d" %couti)
-            tdSql.execute("drop database if exists db%d" %couti)
-            print("create database if not exists db%d replica 1 duration 300" %couti)
-            tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
-            tdSql.execute("use db%d" %couti)
-            tdSql.execute(
-            '''create table stb1
-            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
-            tags (t1 int)
-            '''
-            )
-            tdSql.execute(
-                '''
-                create table t1
-                (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
-                '''
-            )
-            for i in range(4):
-                tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
-
-
     def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole):
         tdLog.printNoPrefix("======== test case 1: ")
         paraDict = {'dbName':     'db',
@@ -143,7 +118,8 @@ class TDTestCase:
         threads=[]
         for i in range(restartNumbers):
             dbNameIndex = '%s%d'%(paraDict["dbName"],i)
-            threads.append(threading.Thread(target=clusterComCreate.create_databases, args=(tdSql, dbNameIndex,paraDict["dbNumbers"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])))
+            newTdSql=tdCom.newTdSql()
+            threads.append(threading.Thread(target=clusterComCreate.create_databases, args=(newTdSql, dbNameIndex,paraDict["dbNumbers"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])))
 
         for tr in threads:
             tr.start()

tests/system-test/6-cluster/clusterCommonCheck.py

@@ -39,6 +39,7 @@ class ClusterComCheck:
 
     def checkDnodes(self,dnodeNumbers):
         count=0
+        # print(tdSql)
         while count < 5:
             tdSql.query("show dnodes")
             # tdLog.debug(tdSql.queryResult)
@@ -85,7 +86,7 @@ class ClusterComCheck:
                             tdLog.debug("check %s_%d that status is ready "%(dbNameIndex,j))      
                         else:
                             continue
-            print(query_status)
+            # print(query_status)
             count+=1
             if query_status == dbNumbers:
                 tdLog.success("we find cluster with %d dnode and check all databases  are ready within 5s! " %dbNumbers)

tests/system-test/6-cluster/clusterCommonCreate.py

@@ -125,7 +125,6 @@ class ClusterComCreate:
 
     def create_databases(self,tsql,dbNameIndex,dbNumbers,dropFlag=1,vgroups=4,replica=1):
         for i in range(dbNumbers):
-            print(dbNumbers)
             if dropFlag == 1:
                 tsql.execute("drop database if exists %s_%d"%(dbNameIndex,i))
             tsql.execute("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica))

tests/system-test/7-tmq/stbFilter.py

@@ -20,18 +20,54 @@ class TDTestCase:
         tdSql.init(conn.cursor())
         #tdSql.init(conn.cursor(), logSql)  # output sql.txt file
 
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   100,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1}
+
+        tmqCom.initConsumerTable()
+        tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica'])
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        return
+
     def tmqCase1(self):
         tdLog.printNoPrefix("======== test case 1: ")
-        paraDict = {'dbName':     'db1',
+        paraDict = {'dbName':     'dbt',
                     'dropFlag':   1,
                     'event':      '',
                     'vgroups':    4,
+                    'replica':    1,
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}],
                     'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
                     'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
                     'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   100,
@@ -43,13 +79,6 @@ class TDTestCase:
         topicNameList = ['topic1', 'topic2', 'topic3']
         expectRowsList = []
         tmqCom.initConsumerTable()
-        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
-        tdLog.info("create stb")
-        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        tdLog.info("create ctb")
-        tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
-        tdLog.info("insert data")
-        tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
         
         tdLog.info("create topics from stb with filter")
         queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 4 == 0" %(paraDict['dbName'], paraDict['stbName'])
@@ -122,9 +151,9 @@ class TDTestCase:
         tdLog.info("wait the consume result") 
         expectRows = 1
         resultList = tmqCom.selectConsumeResult(expectRows)
-        # if expectRowsList[2] != resultList[0]:
-        #     tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
-        #     tdLog.exit("2 tmq consume rows error!")
+        if expectRowsList[2] != resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
+            tdLog.exit("2 tmq consume rows error!")
 
         time.sleep(10)        
         for i in range(len(topicNameList)):
@@ -134,16 +163,18 @@ class TDTestCase:
 
     def tmqCase2(self):
         tdLog.printNoPrefix("======== test case 2: ")
-        paraDict = {'dbName':     'db2',
+        paraDict = {'dbName':     'dbt',
                     'dropFlag':   1,
                     'event':      '',
                     'vgroups':    4,
+                    'replica':    1,
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
                     'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}],
                     'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
                     'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
                     'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   100,
@@ -155,13 +186,6 @@ class TDTestCase:
         topicNameList = ['topic1', 'topic2', 'topic3']
         expectRowsList = []
         tmqCom.initConsumerTable()
-        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
-        tdLog.info("create stb")
-        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        tdLog.info("create ctb")
-        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'])
-        tdLog.info("insert data")
-        tmqCom.insert_data_1(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
 
         tdLog.info("create topics from stb with filter")
         # sqlString = "create topic %s as select ts, sin(c1), pow(c2,3) from %s.%s where c2 >= 0" %(topicNameList[0], paraDict['dbName'], paraDict['stbName'])
@@ -247,6 +271,7 @@ class TDTestCase:
 
     def run(self):
         tdSql.prepare()
+        self.prepareTestEnv()
         self.tmqCase1()
         self.tmqCase2()
 

tests/system-test/7-tmq/tmqCommon.py

@@ -170,33 +170,42 @@ class TMQCom:
         tdLog.debug("complete to create database %s"%(dbName))
         return
 
+    # self.create_stable() and self.create_ctable() and self.insert_data_interlaceByMultiTbl() : The three functions are matched
+    # schema: (ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32))
     def create_stable(self,tsql, dbName,stbName):
-        tsql.execute("create table if not exists %s.%s (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbName, stbName))
+        schemaString = "(ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32))"
+        tsql.execute("create table if not exists %s.%s %s"%(dbName, stbName, schemaString))
         tdLog.debug("complete to create %s.%s" %(dbName, stbName))
         return
 
     def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0):
-        tsql.execute("use %s" %dbName)
+        # tsql.execute("use %s" %dbName)
         pre_create = "create table"
         sql = pre_create
         #tdLog.debug("doing create one  stable %s and %d  child table in %s  ..." %(stbname, count ,dbname))
+        batchNum = 10
+        tblBatched  = 0
         for i in range(ctbNum):
-            tagValue = 'beijing'
+            tagBinaryValue = 'beijing'
             if (i % 2 == 0):
-                tagValue = 'shanghai'
+                tagBinaryValue = 'shanghai'
             elif (i % 3 == 0):
-                tagValue = 'changsha'
+                tagBinaryValue = 'changsha'
             
-            sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i+ctbStartIdx,stbName,i+ctbStartIdx+1, tagValue)
-            if (i > 0) and (i%100 == 0):
+            sql += " %s.%s%d using %s.%s tags(%d, %d, %d, '%s', '%s')"%(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,tagBinaryValue,tagBinaryValue)
+            tblBatched += 1
+            if (i == ctbNum-1 ) or (tblBatched == batchNum):
                 tsql.execute(sql)
+                tblBatched = 0
                 sql = pre_create
+
         if sql != pre_create:
             tsql.execute(sql)
         
-        tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName))
+        tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName))
         return    
 
+    # schema: (ts timestamp, c1 int, c2 binary(16))
     def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
@@ -208,11 +217,14 @@ class TMQCom:
             startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
+            rowsBatched = 0
             sql += " %s%d values "%(stbName,i)
             for j in range(rowsPerTbl):
                 sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
-                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+                rowsBatched += 1
+                if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
                     tsql.execute(sql)
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
                         sql = "insert into %s%d values " %(stbName,i)
                     else:
@@ -224,6 +236,7 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return        
 
+    # schema: (ts timestamp, c1 int, c2 int, c3 binary(16))
     def insert_data_1(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
@@ -234,14 +247,17 @@ class TMQCom:
         startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
+            rowsBatched = 0
             sql += " %s%d values "%(ctbPrefix,i)
             for j in range(rowsPerTbl):
                 if (j % 2 == 0):
                     sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, j, j)
                 else:
                     sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, -j, j)
-                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+                rowsBatched += 1
+                if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
                     tsql.execute(sql)
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
                         sql = "insert into %s%d values " %(ctbPrefix,i)
                     else:
@@ -253,6 +269,7 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return
 
+    # schema: (ts timestamp, c1 int, c2 int, c3 binary(16), c4 timestamp)
     def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,ctbStartIdx=0):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
@@ -263,14 +280,17 @@ class TMQCom:
         startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
+            rowsBatched = 0
             sql += " %s%d values "%(ctbPrefix,i+ctbStartIdx)
             for j in range(rowsPerTbl):
                 if (j % 2 == 0):
                     sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, j, j)
                 else:
                     sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, -j, j)
-                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+                rowsBatched += 1
+                if (rowsBatched == batchNum) or (j == rowsPerTbl - 1):
                     tsql.execute(sql)
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
                         sql = "insert into %s%d values " %(ctbPrefix,i+ctbStartIdx)
                     else:
@@ -282,7 +302,8 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return
 
-    def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0):
+    # schema: (ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32))
+    def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0,ctbStartIdx=0):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
         pre_insert = "insert into "
@@ -300,12 +321,19 @@ class TMQCom:
         rowsOfCtb = 0
         while rowsOfCtb < rowsPerTbl:
             for i in range(ctbNum):
-                sql += " %s.%s_%d values "%(dbName,ctbPrefix,i)
+                sql += " %s.%s%d values "%(dbName,ctbPrefix,i+ctbStartIdx)
+                rowsBatched = 0
                 for k in range(batchNum):
-                    sql += "(%d, %d, 'tmqrow_%d') "%(startTs + ctbDict[i], ctbDict[i], ctbDict[i])
+                    if (k % 2 == 0):
+                        sql += "(%d, %d, %d, %d, 'binary_%d', 'nchar_%d', now) "%(startTs+ctbDict[i], ctbDict[i],ctbDict[i], ctbDict[i],i+ctbStartIdx,k)
+                    else:
+                        sql += "(%d, %d, %d, %d, 'binary_%d', 'nchar_%d', now) "%(startTs+ctbDict[i],-ctbDict[i],ctbDict[i],-ctbDict[i],i+ctbStartIdx,k)
+
+                    rowsBatched += 1
                     ctbDict[i] += 1
-                    if (0 == ctbDict[i]%batchNum) or (ctbDict[i] == rowsPerTbl):
+                    if (rowsBatched == batchNum) or (ctbDict[i] == rowsPerTbl):
                         tsql.execute(sql)
+                        rowsBatched = 0
                         sql = "insert into "
                         break
             rowsOfCtb = ctbDict[0]
@@ -313,7 +341,18 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return
 
-    def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0):
+    def threadFunctionForInsertByInterlace(self, **paraDict):
+        # create new connector for new tdSql instance in my thread
+        newTdSql = tdCom.newTdSql()
+        self.insert_data_interlaceByMultiTbl(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"],paraDict["ctbStartIdx"])
+        return
+
+    def asyncInsertDataByInterlace(self, paraDict):
+        pThread = threading.Thread(target=self.threadFunctionForInsertByInterlace, kwargs=paraDict)
+        pThread.start()
+        return pThread
+
+    def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0,ctbStartIdx=0):
         tdLog.debug("start to insert data wiht auto create child table ............")
         tsql.execute("use %s" %dbName)
         pre_insert = "insert into "
@@ -324,17 +363,17 @@ class TMQCom:
             startTs = int(round(t * 1000))
 
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
-        rowsOfSql = 0        
+        rowsBatched = 0        
         for i in range(ctbNum):
-            sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i,dbName,stbName,i)
+            sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i)
             for j in range(rowsPerTbl):
                 sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
-                rowsOfSql += 1
-                if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)):
+                rowsBatched += 1
+                if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
                     tsql.execute(sql)
-                    rowsOfSql = 0
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
-                        sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i,dbName,stbName,i)
+                        sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i)
                     else:
                         sql = "insert into "
         #end sql

tests/system-test/7-tmq/tmqConsFromTsdb.py

@@ -16,7 +16,7 @@ sys.path.append("./7-tmq")
 from tmqCommon import *
 
 class TDTestCase:
-    def __int__(self):
+    def __init__(self):
         self.vgroups    = 1
         self.ctbNum     = 10
         self.rowsPerTbl = 10000
@@ -25,8 +25,8 @@ class TDTestCase:
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor(), False)
 
-    def tmqCase1(self):
-        tdLog.printNoPrefix("======== test case 1: ")
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
         paraDict = {'dbName':     'dbt',
                     'dropFlag':   1,
                     'event':      '',
@@ -34,10 +34,11 @@ class TDTestCase:
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
                     'ctbPrefix':  'ctb',
-                    'ctbNum':     1,
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   10,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
@@ -50,19 +51,52 @@ class TDTestCase:
         paraDict['ctbNum'] = self.ctbNum
         paraDict['rowsPerTbl'] = self.rowsPerTbl
         
-        topicNameList = ['topic1']
-        expectRowsList = []
         tmqCom.initConsumerTable()
         tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
         tdLog.info("create stb")
-        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
         tdLog.info("create ctb")
-        tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
         tdLog.info("insert data")
-        tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
         
+        tdLog.info("restart taosd to ensure that the data falls into the disk")
         tdDnodes.stop(1)
         tdDnodes.start(1)
+        return
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
         
         tdLog.info("create topics from stb with filter")
         queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
@@ -111,10 +145,11 @@ class TDTestCase:
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
                     'ctbPrefix':  'ctb',
-                    'ctbNum':     1,
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   10,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
@@ -130,16 +165,6 @@ class TDTestCase:
         topicNameList = ['topic1']
         expectRowsList = []
         tmqCom.initConsumerTable()
-        # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
-        # tdLog.info("create stb")
-        # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        # tdLog.info("create ctb")
-        # tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
-        # tdLog.info("insert data")
-        # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
-        
-        # tdDnodes.stop(1)
-        # tdDnodes.start(1)
         
         tdLog.info("create topics from stb with filter")
         queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
@@ -200,89 +225,9 @@ class TDTestCase:
 
         tdLog.printNoPrefix("======== test case 2 end ...... ")
 
-    def tmqCase3(self):
-        tdLog.printNoPrefix("======== test case 3: ")
-        paraDict = {'dbName':     'dbt',
-                    'dropFlag':   1,
-                    'event':      '',
-                    'vgroups':    1,
-                    'stbName':    'stb',
-                    'colPrefix':  'c',
-                    'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'ctbPrefix':  'ctb',
-                    'ctbNum':     1,
-                    'rowsPerTbl': 10000,
-                    'batchNum':   10,
-                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    'pollDelay':  -1,
-                    'showMsg':    1,
-                    'showRow':    1,
-                    'snapshot':   1}
-
-        paraDict['vgroups'] = self.vgroups
-        paraDict['ctbNum'] = self.ctbNum
-        paraDict['rowsPerTbl'] = self.rowsPerTbl
-        
-        topicNameList = ['topic1']
-        expectRowsList = []
-        tmqCom.initConsumerTable()
-        # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
-        # tdLog.info("create stb")
-        # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        # tdLog.info("create ctb")
-        # tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
-        # tdLog.info("insert data")
-        # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
-        
-        # tdDnodes.stop(1)
-        # tdDnodes.start(1)
-        
-        tdLog.info("create topics from stb with filter")
-        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
-        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
-        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
-        tdLog.info("create topic sql: %s"%sqlString)
-        tdSql.execute(sqlString)
-        tdSql.query(queryString)        
-        expectRowsList.append(tdSql.getRows())
-        totalRowsInserted = expectRowsList[0]
-
-        # init consume info, and start tmq_sim, then check consume result
-        tdLog.info("insert consume info to consume processor")
-        consumerId   = 3
-        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3)
-        topicList    = topicNameList[0]
-        ifcheckdata  = 1
-        ifManualCommit = 1
-        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
-        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-
-        consumerId   = 4
-        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3)
-        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-
-        tdLog.info("start consume processor 0")
-        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
-        tdLog.info("wait the consume result") 
-        
-        expectRows = 2
-        resultList = tmqCom.selectConsumeResult(expectRows)
-        actConsumeTotalRows = resultList[0] + resultList[1]
-        
-        if not (totalRowsInserted == actConsumeTotalRows):
-            tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
-            tdLog.exit("%d tmq consume rows error!"%consumerId)
-
-        time.sleep(10)        
-        for i in range(len(topicNameList)):
-            tdSql.query("drop topic %s"%topicNameList[i])
-
-        tdLog.printNoPrefix("======== test case 3 end ...... ")
-
     def run(self):
         tdSql.prepare()
+        self.prepareTestEnv()
         self.tmqCase1()
         self.tmqCase2()
 

tests/system-test/7-tmq/tmqConsFromTsdb1.py

+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def __init__(self):
+        self.vgroups    = 1
+        self.ctbNum     = 10
+        self.rowsPerTbl = 10000
+        
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        tdLog.info("restart taosd to ensure that the data falls into the disk")
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        return
+
+    def tmqCase3(self):
+        tdLog.printNoPrefix("======== test case 3: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+        totalRowsInserted = expectRowsList[0]
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 3
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3)
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        consumerId   = 4
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3)
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor 0")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result") 
+        
+        expectRows = 2
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        actConsumeTotalRows = resultList[0] + resultList[1]
+        
+        if not (totalRowsInserted == actConsumeTotalRows):
+            tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        time.sleep(10)        
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 3 end ...... ")
+
+    def tmqCase4(self):
+        tdLog.printNoPrefix("======== test case 4: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+        totalRowsInserted = expectRowsList[0]
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 5
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor 0")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        
+        tdLog.info("wait commit notify")
+        tmqCom.getStartCommitNotifyFromTmqsim()
+
+        tdLog.info("pkill consume processor")
+        tdCom.killProcessor("tmq_sim")
+        
+        # time.sleep(10)
+        
+        # reinit consume info, and start tmq_sim, then check consume result
+        tmqCom.initConsumerTable()
+        consumerId   = 6
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        tdLog.info("start consume processor 1")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result")
+        
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        actConsumeTotalRows = resultList[0]
+        
+        if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted):
+            tdLog.info("act consume rows: %d"%(actConsumeTotalRows))            
+            tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        time.sleep(10)        
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 4 end ...... ")
+
+    def run(self):
+        tdSql.prepare()
+        self.prepareTestEnv()
+        self.tmqCase3()
+        self.tmqCase4()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/fulltest.sh

@@ -18,7 +18,8 @@ python3 ./test.py -f 0-others/fsync.py
 python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py
 python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py
 python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py
-# BUG python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py
+python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py
+python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py 
 python3 ./test.py -f 1-insert/alter_stable.py
 python3 ./test.py -f 1-insert/alter_table.py
 python3 ./test.py -f 1-insert/insertWithMoreVgroup.py
@@ -89,7 +90,7 @@ python3 ./test.py -f 2-query/query_cols_tags_and_or.py
 # python3 ./test.py -f 2-query/nestedQuery_str.py
 
 python3 ./test.py -f 2-query/avg.py
-python3 ./test.py -f 2-query/elapsed.py
+#python3 ./test.py -f 2-query/elapsed.py
 python3 ./test.py -f 2-query/csum.py
 python3 ./test.py -f 2-query/mavg.py
 python3 ./test.py -f 2-query/diff.py
@@ -121,7 +122,7 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3
 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3
 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 
-# python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py  -N 5 -M 3
+python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py  -N 5 -M 3
 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 5 -M 3 
 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py  -N 5 -M 3 
 # python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py  -N 5 -M 3
@@ -154,3 +155,5 @@ python3 ./test.py -f 7-tmq/tmqUdf.py
 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py
 python3 ./test.py -f 7-tmq/tmqShow.py
 python3 ./test.py -f 7-tmq/tmqAlterSchema.py
+python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py
+python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py

tests/system-test/test.py

@@ -63,8 +63,9 @@ if __name__ == "__main__":
     mnodeNums = 0
     updateCfgDict = {}
     execCmd = ""
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:', [
-        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums'])
+    queryPolicy = 1
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:', [
+        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy'])
     for key, value in opts:
         if key in ['-h', '--help']:
             tdLog.printNoPrefix(
@@ -82,6 +83,7 @@ if __name__ == "__main__":
             tdLog.printNoPrefix('-e eval str to run')
             tdLog.printNoPrefix('-N create dnodes numbers in clusters')
             tdLog.printNoPrefix('-M create mnode numbers in clusters')
+            tdLog.printNoPrefix('-Q set queryPolicy in one dnode')
 
             sys.exit(0)
 
@@ -138,6 +140,9 @@ if __name__ == "__main__":
         if key in ['-M', '--mnodeNums']:
             mnodeNums = value
 
+        if key in ['-Q', '--queryPolicy']:
+            queryPolicy = value
+
     if not execCmd == "":
         tdDnodes.init(deployPath)
         print(execCmd)
@@ -276,6 +281,22 @@ if __name__ == "__main__":
             tdDnodes.deploy(1,updateCfgDict)
             tdDnodes.start(1)
             tdCases.logSql(logSql)
+            if queryPolicy != 1:
+                queryPolicy=int(queryPolicy)
+                conn = taos.connect(
+                host,
+                config=tdDnodes.getSimCfgPath())
+                tdSql.init(conn.cursor())
+                tdSql.execute("create qnode on dnode 1")
+                tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy)     
+                tdSql.query("show local variables;")
+                for i in range(tdSql.queryRows):
+                    if tdSql.queryResult[i][0] == "queryPolicy" :
+                        if int(tdSql.queryResult[i][1]) == int(queryPolicy):
+                            tdLog.success('alter queryPolicy to %d successfully'%queryPolicy)
+                        else :
+                            tdLog.debug(tdSql.queryResult)
+                            tdLog.exit("alter queryPolicy to  %d failed"%queryPolicy)      
         else :
             tdLog.debug("create an cluster  with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
             dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums)

tests/test/c/tmqSim.c

@@ -90,6 +90,7 @@ typedef struct {
   int32_t     consumeDelay;  // unit s
   int32_t     numOfThread;
   int32_t     useSnapshot;
+  int64_t     nowTime;
   SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT];
 } SConfInfo;
 
@@ -198,6 +199,8 @@ void parseArgument(int32_t argc, char* argv[]) {
   g_stConfInfo.saveRowFlag = 0;
   g_stConfInfo.consumeDelay = 5;
 
+  g_stConfInfo.nowTime = taosGetTimestampMs();
+
   for (int32_t i = 1; i < argc; i++) {
     if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
       printHelp();
@@ -510,10 +513,8 @@ static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) {}
 int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) {
   char sqlStr[1024] = {0};
 
-  int64_t now = taosGetTimestampMs();
-
   // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int
-  sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, now, cmdId,
+  sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), cmdId,
           pInfo->consumerId);
 
   taos_query_a(pInfo->taos, sqlStr, appNothing, NULL);
@@ -588,12 +589,10 @@ void build_topic_list(SThreadInfo* pInfo) {
 
 int32_t saveConsumeResult(SThreadInfo* pInfo) {
   char sqlStr[1024] = {0};
-
-  int64_t now = taosGetTimestampMs();
-
   // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int
   sprintf(sqlStr, "insert into %s.consumeresult values (%" PRId64 ", %d, %" PRId64 ", %" PRId64 ", %d)",
-          g_stConfInfo.cdbName, now, pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult);
+          g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), pInfo->consumerId, pInfo->consumeMsgCnt,
+          pInfo->consumeRowCnt, pInfo->checkresult);
 
   char tmpString[128];
   taosFprintfFile(g_fp, "%s, consume id %d result: %s\n", getCurrentTimeString(tmpString), pInfo->consumerId, sqlStr);
@@ -655,9 +654,9 @@ void loop_consume(SThreadInfo* pInfo) {
 
       int64_t currentPrintTime = taosGetTimestampMs();
       if (currentPrintTime - lastPrintTime > 10 * 1000) {
-		  taosFprintfFile(g_fp,	
-		  	              "consumer id %d has currently poll total msgs: %" PRId64 ", period rate: %.3f msgs/second\n", 
-		  	              pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0/(currentPrintTime - lastPrintTime));
+        taosFprintfFile(
+            g_fp, "consumer id %d has currently poll total msgs: %" PRId64 ", period rate: %.3f msgs/second\n",
+            pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / (currentPrintTime - lastPrintTime));
         lastPrintTime = currentPrintTime;
         lastTotalMsgs = totalMsgs;
       }
@@ -696,6 +695,7 @@ void* consumeThreadFunc(void* param) {
   pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0);
   if (pInfo->taos == NULL) {
     taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n");
+    ASSERT(0);
     return NULL;
   }