Merge branch '3.0' of github.com:taosdata/TDengine into test/chr/TD-14699

cmake/cmake.define

@@ -98,12 +98,12 @@ ELSE ()
     ENDIF ()
 
     IF (${SANITIZER} MATCHES "true")
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
 	MESSAGE(STATUS "Will compile with Address Sanitizer!")
     ELSE ()
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")
     ENDIF ()
 
     MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")

include/common/tcommon.h

@@ -56,7 +56,8 @@ typedef enum EStreamType {
   STREAM_CLEAR,
   STREAM_INVALID,
   STREAM_GET_ALL,
-  STREAM_DELETE,
+  STREAM_DELETE_RESULT,
+  STREAM_DELETE_DATA,
   STREAM_RETRIEVE,
   STREAM_PULL_DATA,
   STREAM_PULL_OVER,

include/libs/qcom/query.h

@@ -65,7 +65,7 @@ typedef struct SQueryExecRes {
 } SQueryExecRes;
 
 typedef struct SIndexMeta {
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 

include/os/os.h

@@ -41,7 +41,6 @@ extern "C" {
 #include <sys/types.h>
 #include <termios.h>
 #include <sys/statvfs.h>
-#include <sys/prctl.h>
 #include <sys/shm.h>
 #include <sys/wait.h>
 

include/os/osAtomic.h

@@ -63,7 +63,7 @@ int8_t  atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
-void   *atomic_add_fetch_ptr(void *ptr, void *val);
+void   *atomic_add_fetch_ptr(void *ptr, int64_t val);
 int8_t  atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
@@ -73,7 +73,7 @@ int8_t  atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
-void   *atomic_sub_fetch_ptr(void *ptr, void *val);
+void   *atomic_sub_fetch_ptr(void *ptr, int64_t val);
 int8_t  atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);

include/os/osMath.h

@@ -20,6 +20,12 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following sectio
+#ifndef ALLOW_FORBID_FUNC
+#define qsort  QSORT_FUNC_TAOS_FORBID
+#endif
+
 #define TPOW2(x) ((x) * (x))
 #define TABS(x) ((x) > 0 ? (x) : -(x))
 

include/os/osSemaphore.h

@@ -24,7 +24,9 @@ extern "C" {
 
 #if defined(_TD_DARWIN_64)
 
-typedef struct tsem_s *tsem_t;
+// typedef struct tsem_s *tsem_t;
+typedef struct bosal_sem_t *tsem_t;
+
 
 int tsem_init(tsem_t *sem, int pshared, unsigned int value);
 int tsem_wait(tsem_t *sem);
@@ -51,11 +53,11 @@ int tsem_timewait(tsem_t *sim, int64_t nanosecs);
 //  #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock)
 //  #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock)
 
-#define TdThreadSpinlock               TdThreadMutex
-#define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
-#define taosThreadSpinDestroy(lock)    taosThreadMutexDestroy(lock)
-#define taosThreadSpinLock(lock)       taosThreadMutexLock(lock)
-#define taosThreadSpinUnlock(lock)     taosThreadMutexUnlock(lock)
+// #define TdThreadSpinlock               TdThreadMutex
+// #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
+// #define taosThreadSpinDestroy(lock)    taosThreadMutexDestroy(lock)
+// #define taosThreadSpinLock(lock)       taosThreadMutexLock(lock)
+// #define taosThreadSpinUnlock(lock)     taosThreadMutexUnlock(lock)
 #endif
 
 bool    taosCheckPthreadValid(TdThread thread);

include/os/osSocket.h

@@ -64,7 +64,6 @@
 #include <osEok.h>
 #else
 #include <netinet/in.h>
-#include <sys/epoll.h>
 #endif
 #endif
 
@@ -77,15 +76,12 @@ typedef int socklen_t;
 #define TAOS_EPOLL_WAIT_TIME 100
 typedef SOCKET eventfd_t;
 #define eventfd(a, b)      -1
-#define EpollClose(pollFd) epoll_close(pollFd)
 #ifndef EPOLLWAKEUP
 #define EPOLLWAKEUP (1u << 29)
 #endif
 #elif defined(_TD_DARWIN_64)
 #define TAOS_EPOLL_WAIT_TIME 500
 typedef int32_t SOCKET;
-typedef SOCKET  EpollFd;
-#define EpollClose(pollFd)   epoll_close(pollFd)
 #else
 #define TAOS_EPOLL_WAIT_TIME 500
 typedef int32_t SOCKET;
@@ -122,14 +118,6 @@ typedef SOCKET  EpollFd;
 typedef int32_t  SocketFd;
 typedef SocketFd EpollFd;
 
-typedef struct TdSocket {
-#if SOCKET_WITH_LOCK
-  TdThreadRwlock rwlock;
-#endif
-  int      refId;
-  SocketFd fd;
-} * TdSocketPtr, TdSocket;
-
 typedef struct TdSocketServer *TdSocketServerPtr;
 typedef struct TdSocket *      TdSocketPtr;
 typedef struct TdEpoll *       TdEpollPtr;
@@ -181,11 +169,6 @@ void        taosSetMaskSIGPIPE();
 uint32_t    taosInetAddr(const char *ipAddr);
 const char *taosInetNtoa(struct in_addr ipInt);
 
-TdEpollPtr taosCreateEpoll(int32_t size);
-int32_t    taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
-int32_t    taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
-int32_t    taosCloseEpoll(TdEpollPtr *ppEpoll);
-
 #ifdef __cplusplus
 }
 #endif

include/os/osString.h

@@ -67,7 +67,7 @@ bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs
 int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
 TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
 bool    taosValidateEncodec(const char *encodec);
-int32_t taosHexEncode(const char *src, char *dst, int32_t len);
+int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len);
 int32_t taosHexDecode(const char *src, char *dst, int32_t len);
 
 int32_t taosWcharWidth(TdWchar wchar);

include/os/osThread.h

@@ -188,27 +188,27 @@ int32_t taosThreadJoin(TdThread thread, void **valuePtr);
 int32_t taosThreadKeyCreate(TdThreadKey * key, void(*destructor)(void *));
 int32_t taosThreadKeyDelete(TdThreadKey key);
 int32_t taosThreadKill(TdThread thread, int32_t sig);
-int32_t taosThreadMutexConsistent(TdThreadMutex* mutex);
+// int32_t taosThreadMutexConsistent(TdThreadMutex* mutex);
 int32_t taosThreadMutexDestroy(TdThreadMutex * mutex);
 int32_t taosThreadMutexInit(TdThreadMutex * mutex, const TdThreadMutexAttr * attr);
 int32_t taosThreadMutexLock(TdThreadMutex * mutex);
-int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime);
+// int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime);
 int32_t taosThreadMutexTryLock(TdThreadMutex * mutex);
 int32_t taosThreadMutexUnlock(TdThreadMutex * mutex);
 int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr * attr);
 int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr * attr, int32_t *pshared);
-int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust);
+// int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust);
 int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr * attr, int32_t *kind);
 int32_t taosThreadMutexAttrInit(TdThreadMutexAttr * attr);
 int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr * attr, int32_t pshared);
-int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust);
+// int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust);
 int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr * attr, int32_t kind);
 int32_t taosThreadOnce(TdThreadOnce * onceControl, void(*initRoutine)(void));
 int32_t taosThreadRwlockDestroy(TdThreadRwlock * rwlock);
 int32_t taosThreadRwlockInit(TdThreadRwlock * rwlock, const TdThreadRwlockAttr * attr);
 int32_t taosThreadRwlockRdlock(TdThreadRwlock * rwlock);
-int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
-int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
+// int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
+// int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
 int32_t taosThreadRwlockTryRdlock(TdThreadRwlock * rwlock);
 int32_t taosThreadRwlockTryWrlock(TdThreadRwlock * rwlock);
 int32_t taosThreadRwlockUnlock(TdThreadRwlock * rwlock);

include/util/taoserror.h

@@ -623,6 +623,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_RSMA_INVALID_ENV               TAOS_DEF_ERROR_CODE(0, 0x3150)
 #define TSDB_CODE_RSMA_INVALID_STAT              TAOS_DEF_ERROR_CODE(0, 0x3151)
 #define TSDB_CODE_RSMA_QTASKINFO_CREATE          TAOS_DEF_ERROR_CODE(0, 0x3152)
+#define TSDB_CODE_RSMA_FILE_CORRUPTED            TAOS_DEF_ERROR_CODE(0, 0x3153)
 
 //index
 #define TSDB_CODE_INDEX_REBUILDING               TAOS_DEF_ERROR_CODE(0, 0x3200)

include/util/types.h

@@ -83,6 +83,7 @@ typedef uint16_t VarDataLenT;  // maxVarDataLen: 32767
 
 #define varDataLen(v) ((VarDataLenT *)(v))[0]
 #define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE)
+#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
 
 #define NCHAR_WIDTH_TO_BYTES(n)  ((n) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE)
 

source/common/src/tdatablock.c

@@ -1259,6 +1259,7 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
 
   dst->info.rows = src->info.rows;
   dst->info.window = src->info.window;
+  dst->info.type = src->info.type;
   return TSDB_CODE_SUCCESS;
 }
 

source/common/src/ttime.c

@@ -589,7 +589,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec
       return TSDB_CODE_FAILED;
     }
     newColData[len] = 0;
-    int32_t ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, tsDaylight);
+    int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight);
     if (ret != TSDB_CODE_SUCCESS) {
       taosMemoryFree(newColData);
       return ret;

source/common/src/ttszip.c

@@ -845,11 +845,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
 
   int64_t offset = getDataStartOffset();
   int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
-#if defined(_TD_DARWIN_64)
-  int64_t written = taosFSendFile(pDestBuf->pFile->fp, pSrcBuf->pFile->fp, &offset, size);
-#else
   int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size);
-#endif
 
   if (written == -1 || written != size) {
     return -1;

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -21,7 +21,7 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) {
 
   taosThreadRwlockRdlock(&pMgmt->lock);
   taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
-  if (pVnode == NULL) {
+  if (pVnode == NULL || pVnode->dropped) {
     terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
   } else {
     int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
@@ -81,16 +81,18 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
 
   vmReleaseVnode(pMgmt, pVnode);
   while (pVnode->refCount > 0) taosMsleep(10);
+  dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId);
+
   while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
+  dTrace("vgId:%d, vnode-fetch queue is empty", pVnode->vgId);
 
   vmFreeQueue(pMgmt, pVnode);
   vnodeClose(pVnode->pImpl);
   pVnode->pImpl = NULL;
-
   dDebug("vgId:%d, vnode is closed", pVnode->vgId);
 
   if (pVnode->dropped) {

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -146,8 +146,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
-    dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(),
-            TMSG_INFO(pMsg->msgType));
+    dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, msgtype:%s qtype:%d", pHead->vgId, pMsg,
+            terrstr(), TMSG_INFO(pMsg->msgType), qtype);
     return terrno != 0 ? terrno : -1;
   }
 

source/dnode/mnode/impl/inc/mndDef.h

@@ -556,7 +556,7 @@ typedef struct {
   int64_t uid;
   int8_t  status;
   // config
-  int8_t  dropPolicy;
+  int8_t  igExpired;
   int8_t  trigger;
   int64_t triggerParam;
   int64_t watermark;

source/dnode/mnode/impl/inc/mndInt.h

@@ -87,7 +87,7 @@ typedef struct {
 } STelemMgmt;
 
 typedef struct {
-  sem_t    syncSem;
+  tsem_t    syncSem;
   int64_t  sync;
   bool     standby;
   SReplica replica;

source/dnode/mnode/impl/inc/mndStream.h

@@ -38,6 +38,8 @@ int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 
+int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/src/mndDb.c

@@ -21,6 +21,7 @@
 #include "mndShow.h"
 #include "mndSma.h"
 #include "mndStb.h"
+#include "mndStream.h"
 #include "mndSubscribe.h"
 #include "mndTopic.h"
 #include "mndTrans.h"
@@ -927,6 +928,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
   if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
+  if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER;
 
@@ -947,7 +949,6 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
   mndTransSetRpcRsp(pTrans, pRsp, rspLen);
 
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
-
   code = 0;
 
 _OVER:

source/dnode/mnode/impl/src/mndDef.c

@@ -28,7 +28,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
   if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
 
-  if (tEncodeI8(pEncoder, pObj->dropPolicy) < 0) return -1;
+  if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->triggerParam) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->watermark) < 0) return -1;
@@ -73,7 +73,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
   if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
 
-  if (tDecodeI8(pDecoder, &pObj->dropPolicy) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->triggerParam) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->watermark) < 0) return -1;

source/dnode/mnode/impl/src/mndSma.c

@@ -523,6 +523,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
   streamObj.updateTime = streamObj.createTime;
   streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
   streamObj.sourceDbUid = pDb->uid;
+  streamObj.targetDbUid = pDb->uid;
   streamObj.version = 1;
   streamObj.sql = pCreate->sql;
   streamObj.smaId = smaObj.uid;
@@ -854,35 +855,25 @@ _OVER:
 
 int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
   SSdb *pSdb = pMnode->pSdb;
-  SSmaObj *pSma = NULL;
   void *pIter = NULL;
-  SVgObj  *pVgroup = NULL;
-  int32_t  code = -1;
 
   while (1) {
+    SSmaObj *pSma = NULL;
     pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma);
     if (pIter == NULL) break;
 
     if (pSma->dbUid == pDb->uid) {
-      pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId);
-      if (pVgroup == NULL) goto _OVER;
-      if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER;
-      if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER;
-      if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
-      mndReleaseVgroup(pMnode, pVgroup);
-      pVgroup = NULL;
+      if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) {
+        sdbRelease(pSdb, pSma);
+        sdbCancelFetch(pSdb, pSma);
+        return -1;
+      }
     }
 
     sdbRelease(pSdb, pSma);
   }
 
-  code = 0;
-
-_OVER:
-  sdbCancelFetch(pSdb, pIter);
-  sdbRelease(pSdb, pSma);
-  mndReleaseVgroup(pMnode, pVgroup);
-  return code;
+  return 0;
 }
 
 static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) {

source/dnode/mnode/impl/src/mndStream.c

@@ -248,7 +248,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
   pObj->status = 0;
 
   // TODO
-  pObj->dropPolicy = 0;
+  pObj->igExpired = pCreate->igExpired;
   pObj->trigger = pCreate->triggerType;
   pObj->triggerParam = pCreate->maxDelay;
   pObj->watermark = pCreate->watermark;
@@ -301,6 +301,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
       .streamQuery = true,
       .triggerType = pObj->trigger == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : pObj->trigger,
       .watermark = pObj->watermark,
+      .igExpired = pObj->igExpired,
   };
 
   // using ast and param to build physical plan
@@ -673,27 +674,29 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
 
 int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
   SSdb *pSdb = pMnode->pSdb;
-
   void *pIter = NULL;
-  SStreamObj *pStream = NULL;
+
   while (1) {
+    SStreamObj *pStream = NULL;
     pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
     if (pIter == NULL) break;
 
     if (pStream->sourceDbUid == pDb->uid || pStream->targetDbUid == pDb->uid) {
       if (pStream->sourceDbUid != pStream->targetDbUid) {
         sdbRelease(pSdb, pStream);
+        sdbCancelFetch(pSdb, pIter);
+        mError("db:%s, failed to drop stream:%s since sourceDbUid:%" PRId64 " not match with targetDbUid:%" PRId64,
+               pDb->name, pStream->name, pStream->sourceDbUid, pStream->targetDbUid);
+        terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST;
         return -1;
       } else {
         // TODO drop all task on snode
         if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) {
           sdbRelease(pSdb, pStream);
+          sdbCancelFetch(pSdb, pIter);
           return -1;
         }
       }
-    } else {
-      sdbRelease(pSdb, pStream);
-      continue;
     }
 
 #if 0

source/dnode/mnode/impl/src/mndTrans.c

@@ -90,7 +90,7 @@ static int32_t mndTransGetActionsSize(SArray *pArray) {
   for (int32_t i = 0; i < actionNum; ++i) {
     STransAction *pAction = taosArrayGet(pArray, i);
     if (pAction->actionType == TRANS_ACTION_RAW) {
-      rawDataLen += (sdbGetRawTotalSize(pAction->pRaw) + sizeof(int32_t));
+      rawDataLen += (sizeof(STransAction) + sdbGetRawTotalSize(pAction->pRaw));
     } else if (pAction->actionType == TRANS_ACTION_MSG) {
       rawDataLen += (sizeof(STransAction) + pAction->contLen);
     } else {
@@ -105,7 +105,7 @@ static int32_t mndTransGetActionsSize(SArray *pArray) {
 static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
-  int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE;
+  int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE + pTrans->paramLen;
   rawDataLen += mndTransGetActionsSize(pTrans->redoActions);
   rawDataLen += mndTransGetActionsSize(pTrans->undoActions);
   rawDataLen += mndTransGetActionsSize(pTrans->commitActions);
@@ -226,7 +226,8 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
 
 _OVER:
   if (terrno != 0) {
-    mError("trans:%d, failed to encode to raw:%p len:%d since %s", pTrans->id, pRaw, dataPos, terrstr());
+    mError("trans:%d, failed to encode to raw:%p maxlen:%d len:%d since %s", pTrans->id, pRaw, sdbGetRawTotalSize(pRaw),
+           dataPos, terrstr());
     sdbFreeRaw(pRaw);
     return NULL;
   }
@@ -1025,7 +1026,7 @@ static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction
   pTrans->lastAction = pAction->id;
   pTrans->lastMsgType = pAction->msgType;
   pTrans->lastEpset = pAction->epSet;
-  pTrans->lastErrorNo == 0;
+  pTrans->lastErrorNo = 0;
   return 0;
 }
 

source/dnode/vnode/CMakeLists.txt

@@ -31,6 +31,8 @@ target_sources(
     "src/sma/smaEnv.c"
     "src/sma/smaUtil.c"
     "src/sma/smaOpen.c"
+    "src/sma/smaCommit.c"
+    "src/sma/smaSnapshot.c"
     "src/sma/smaRollup.c"
     "src/sma/smaTimeRange.c"
 

source/dnode/vnode/src/inc/sma.h

@@ -92,8 +92,9 @@ enum {
   TASK_TRIGGER_STAT_INIT = 0,
   TASK_TRIGGER_STAT_ACTIVE = 1,
   TASK_TRIGGER_STAT_INACTIVE = 2,
-  TASK_TRIGGER_STAT_CANCELLED = 3,
-  TASK_TRIGGER_STAT_FINISHED = 4,
+  TASK_TRIGGER_STAT_PAUSED = 3,
+  TASK_TRIGGER_STAT_CANCELLED = 4,
+  TASK_TRIGGER_STAT_FINISHED = 5,
 };
 void  tdDestroySmaEnv(SSmaEnv *pSmaEnv);
 void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
@@ -214,25 +215,22 @@ struct STFInfo {
 };
 
 struct STFile {
+  uint8_t   state;
   STFInfo   info;
-  STfsFile  f;
+  char     *fname;
   TdFilePtr pFile;
-  uint8_t   state;
 };
 
-#define TD_TFILE_F(tf)            (&((tf)->f))
 #define TD_TFILE_PFILE(tf)        ((tf)->pFile)
 #define TD_TFILE_OPENED(tf)       (TD_TFILE_PFILE(tf) != NULL)
-#define TD_TFILE_FULL_NAME(tf)    (TD_TFILE_F(tf)->aname)
-#define TD_TFILE_REL_NAME(tf)     (TD_TFILE_F(tf)->rname)
+#define TD_TFILE_FULL_NAME(tf)    ((tf)->fname)
 #define TD_TFILE_OPENED(tf)       (TD_TFILE_PFILE(tf) != NULL)
 #define TD_TFILE_CLOSED(tf)       (!TD_TFILE_OPENED(tf))
 #define TD_TFILE_SET_CLOSED(f)    (TD_TFILE_PFILE(f) = NULL)
 #define TD_TFILE_SET_STATE(tf, s) ((tf)->state = (s))
-#define TD_TFILE_DID(tf)          (TD_TFILE_F(tf)->did)
 
-int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname);
-int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType);
+int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname);
+int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType);
 int32_t tdOpenTFile(STFile *pTFile, int flags);
 int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte);
 int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence);
@@ -244,8 +242,10 @@ int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo);
 int32_t tdUpdateTFileHeader(STFile *pTFile);
 void    tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
 void    tdCloseTFile(STFile *pTFile);
+void    tdDestroyTFile(STFile *pTFile);
 
-void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName);
+void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, int64_t version, char *outputName);
+void tdGetVndDirName(int32_t vgId, const char *dname, char *outputName);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -165,6 +165,9 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
 int32_t smaOpen(SVnode* pVnode);
 int32_t smaCloseEnv(SSma* pSma);
 int32_t smaCloseEx(SSma* pSma);
+int32_t smaPreCommit(SSma* pSma);
+int32_t smaCommit(SSma* pSma);
+int32_t smaPostCommit(SSma* pSma);
 
 int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
 int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
@@ -238,7 +241,7 @@ struct SVnode {
   tsem_t     canCommit;
   int64_t    sync;
   int32_t    syncCount;
-  sem_t      syncSem;
+  tsem_t     syncSem;
   SQHandle*  pQuery;
 };
 

source/dnode/vnode/src/sma/smaCommit.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sma.h"
+
+static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma);
+static int32_t tdProcessRSmaCommitImpl(SSma *pSma);
+static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma);
+
+/**
+ * @brief Only applicable to Rollup SMA
+ *
+ * @param pSma
+ * @return int32_t
+ */
+int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); }
+
+/**
+ * @brief Only applicable to Rollup SMA
+ *
+ * @param pSma
+ * @return int32_t
+ */
+int32_t smaCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); }
+
+/**
+ * @brief Only applicable to Rollup SMA
+ *
+ * @param pSma
+ * @return int32_t
+ */
+int32_t smaPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); }
+
+/**
+ * @brief pre-commit for rollup sma.
+ *  1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED.
+ *  2) perform persist task for qTaskInfo
+ *  3) wait all triggered fetch tasks finished
+ *  4) set trigger stat of rsma timer TASK_TRIGGER_STAT_ACTIVE.
+ *  5) finish
+ *
+ * @param pSma
+ * @return int32_t
+ */
+static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) {
+  SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
+  if (!pSmaEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+
+  // step 1
+  atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED);
+
+  // step 2
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief commit for rollup sma
+ *
+ * @param pSma
+ * @return int32_t
+ */
+static int32_t tdProcessRSmaCommitImpl(SSma *pSma) {
+  SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
+  if (!pSmaEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief post-commit for rollup sma
+ *  1) clean up the outdated qtaskinfo files
+ *
+ * @param pSma
+ * @return int32_t
+ */
+static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) {
+  SVnode *pVnode = pSma->pVnode;
+
+  if (!VND_IS_RSMA(pVnode)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int64_t       committed = pVnode->state.committed;
+  TdDirPtr      pDir = NULL;
+  TdDirEntryPtr pDirEntry = NULL;
+  char          dir[TSDB_FILENAME_LEN];
+  char          bname[TSDB_FILENAME_LEN];
+  const char   *pattern = "^v[0-9]+qtaskinfo\\.ver([0-9]+)?$";
+  regex_t       regex;
+
+  tdGetVndDirName(TD_VID(pVnode), VNODE_RSMA_DIR, dir);
+
+  // Resource allocation and init
+  regcomp(&regex, pattern, REG_EXTENDED);
+
+  if ((pDir = taosOpenDir(dir)) == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    smaWarn("rsma post-commit open dir %s failed since %s", dir, terrstr());
+    return TSDB_CODE_FAILED;
+  }
+
+  regmatch_t regMatch[2];
+  while ((pDirEntry = taosReadDir(pDir)) != NULL) {
+    char *entryName = taosGetDirEntryName(pDirEntry);
+    if (!entryName) {
+      continue;
+    }
+    char *fileName = taosDirEntryBaseName(entryName);
+    int   code = regexec(&regex, bname, 2, regMatch, 0);
+
+    if (code == 0) {
+      // match
+      printf("match 0 = %s\n", (char *)POINTER_SHIFT(fileName, regMatch[0].rm_so));
+      printf("match 1 = %s\n", (char *)POINTER_SHIFT(fileName, regMatch[1].rm_so));
+    } else if (code == REG_NOMATCH) {
+      // not match
+      smaInfo("rsma post-commit not match %s", fileName);
+      continue;
+    } else {
+      // has other error
+      terrno = TAOS_SYSTEM_ERROR(code);
+      smaWarn("rsma post-commit regexec failed since %s", terrstr());
+
+      taosCloseDir(&pDir);
+      regfree(&regex);
+      return TSDB_CODE_FAILED;
+    }
+  }
+  taosCloseDir(&pDir);
+  return TSDB_CODE_SUCCESS;
+}

source/dnode/vnode/src/sma/smaRollup.c

@@ -23,8 +23,7 @@ SSmaMgmt smaMgmt = {
     .smaRef = -1,
 };
 
-typedef enum { TD_QTASK_TMP_F = 0, TD_QTASK_CUR_F } TD_QTASK_FILE_T;
-static const char                *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
+#define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver"
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
 
@@ -37,7 +36,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
 static void    tdRSmaFetchTrigger(void *param, void *tmrId);
 static void    tdRSmaPersistTrigger(void *param, void *tmrId);
 static void   *tdRSmaPersistExec(void *param);
-static void    tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName);
+static void    tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);
 
 static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile);
 static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish);
@@ -88,8 +87,8 @@ struct SRSmaQTaskInfoIter {
   int32_t nBufPos;
 };
 
-static void tdRSmaQTaskInfoGetFName(int32_t vgId, int8_t ftype, char *outputName) {
-  tdGetVndFileName(vgId, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName);
+static void tdRSmaQTaskInfoGetFName(int32_t vgId, int64_t version, char *outputName) {
+  tdGetVndFileName(vgId, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName);
 }
 
 static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) {
@@ -493,7 +492,6 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) {
 }
 
 static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
-  ASSERT(pMsg != NULL);
   SSubmitMsgIter msgIter = {0};
   SSubmitBlk    *pBlock = NULL;
   SSubmitBlkIter blkIter = {0};
@@ -501,19 +499,26 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
 
   terrno = TSDB_CODE_SUCCESS;
 
-  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
+  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) {
+    return -1;
+  }
   while (true) {
-    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
+    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) {
+      return -1;
+    }
 
     if (!pBlock) break;
     tdUidStorePut(pStore, msgIter.suid, NULL);
   }
 
-  if (terrno != TSDB_CODE_SUCCESS) return -1;
+  if (terrno != TSDB_CODE_SUCCESS) {
+    return -1;
+  }
   return 0;
 }
 
 static void tdDestroySDataBlockArray(SArray *pArray) {
+  // TODO
 #if 0
   for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
     SSDataBlock *pDataBlock = taosArrayGet(pArray, i);
@@ -598,18 +603,25 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
 
   pSma = RSMA_INFO_SMA(pItem->pRsmaInfo);
 
-  // if rsma trigger stat in cancelled or finished, not start fetch task anymore
+  // if rsma trigger stat in paused, cancelled or finished, not start fetch task
   int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
-  if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) {
+  switch (rsmaTriggerStat) {
+    case TASK_TRIGGER_STAT_PAUSED:
+    case TASK_TRIGGER_STAT_CANCELLED:
+    case TASK_TRIGGER_STAT_FINISHED: {
       taosReleaseRef(smaMgmt.smaRef, pItem->refId);
       smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled",
                SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
       return;
     }
+    default:
+      break;
+  }
 
   int8_t fetchTriggerStat =
       atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
-  if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) {
+  switch (fetchTriggerStat) {
+    case TASK_TRIGGER_STAT_ACTIVE: {
       smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
                pItem->level, pItem->pRsmaInfo->suid);
 
@@ -620,11 +632,25 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
       tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
 
       tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
-
-  } else {
+    } break;
+    case TASK_TRIGGER_STAT_PAUSED: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
+               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INACTIVE: {
       smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
                SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INIT: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
+               pItem->level, pItem->pRsmaInfo->suid);
+    } break;
+    default: {
+      smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
+              SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+    } break;
   }
+
 _end:
   taosReleaseRef(smaMgmt.smaRef, pItem->refId);
 }
@@ -780,10 +806,10 @@ _err:
 static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
   SVnode *pVnode = pSma->pVnode;
   STFile  tFile = {0};
-  char    qTaskInfoFName[TSDB_FILENAME_LEN];
+  char    qTaskInfoFName[TSDB_FILENAME_LEN] = {0};
 
-  tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_TMP_F, qTaskInfoFName);
-  if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) {
+  tdRSmaQTaskInfoGetFName(TD_VID(pVnode), pVnode->state.committed, qTaskInfoFName);
+  if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
     goto _err;
   }
 
@@ -799,17 +825,20 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
   if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
     tdRSmaQTaskInfoIterDestroy(&fIter);
     tdCloseTFile(&tFile);
+    tdDestroyTFile(&tFile);
     goto _err;
   }
 
   if (tdRSmaQTaskInfoRestore(pSma, &fIter) < 0) {
     tdRSmaQTaskInfoIterDestroy(&fIter);
     tdCloseTFile(&tFile);
+    tdDestroyTFile(&tFile);
     goto _err;
   }
 
   tdRSmaQTaskInfoIterDestroy(&fIter);
   tdCloseTFile(&tFile);
+  tdDestroyTFile(&tFile);
   return TSDB_CODE_SUCCESS;
 _err:
   smaError("rsma restore, qtaskinfo reload failed since %s", terrstr());
@@ -931,19 +960,21 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
   }
 
   if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) {
-    ASSERT(0);
     return TSDB_CODE_FAILED;
   }
 
   if (tdReadTFile(pTFile, pIter->qBuf, nBytes) != nBytes) {
-    ASSERT(0);
     return TSDB_CODE_FAILED;
   }
 
   int32_t infoLen = 0;
   taosDecodeFixedI32(pIter->qBuf, &infoLen);
   if (infoLen > nBytes) {
-    ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE);
+    if (infoLen <= RSMA_QTASKINFO_BUFSIZE) {
+      terrno = TSDB_CODE_RSMA_FILE_CORRUPTED;
+      smaError("iterate rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr());
+      return TSDB_CODE_FAILED;
+    }
     pIter->nAlloc = infoLen;
     void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen);
     if (!pBuf) {
@@ -955,12 +986,10 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
     nBytes = infoLen;
 
     if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) {
-      ASSERT(0);
       return TSDB_CODE_FAILED;
     }
 
     if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) {
-      ASSERT(0);
       return TSDB_CODE_FAILED;
     }
   }
@@ -977,7 +1006,6 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
     // block iter
     bool isFinish = false;
     if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) {
-      ASSERT(0);
       return TSDB_CODE_FAILED;
     }
     if (isFinish) {
@@ -989,6 +1017,7 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
     pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead);
     if (qTaskInfoLenWithHead < RSMA_QTASKINFO_HEAD_LEN) {
       terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+      smaError("restore rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr());
       return TSDB_CODE_FAILED;
     }
 
@@ -1025,22 +1054,16 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
   return TSDB_CODE_SUCCESS;
 }
 
-static void *tdRSmaPersistExec(void *param) {
-  setThreadName("rsma-task-persist");
-  SRSmaStat *pRSmaStat = param;
+static int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) {
   SSma   *pSma = pRSmaStat->pSma;
-  STfs      *pTfs = pSma->pVnode->pTfs;
+  SVnode *pVnode = pSma->pVnode;
   int32_t vid = SMA_VID(pSma);
   int64_t toffset = 0;
   bool    isFileCreated = false;
 
-  if (TASK_TRIGGER_STAT_CANCELLED == atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))) {
-    goto _end;
-  }
-
   void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL);
   if (!infoHash) {
-    goto _end;
+    return TSDB_CODE_SUCCESS;
   }
 
   STFile tFile = {0};
@@ -1074,9 +1097,13 @@ static void *tdRSmaPersistExec(void *param) {
 
       if (!isFileCreated) {
         char qTaskInfoFName[TSDB_FILENAME_LEN];
-        tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_F, qTaskInfoFName);
-        tdInitTFile(&tFile, pTfs, qTaskInfoFName);
-        tdCreateTFile(&tFile, pTfs, true, -1);
+        tdRSmaQTaskInfoGetFName(vid, pSma->pVnode->state.applied, qTaskInfoFName);
+        if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
+          goto _err;
+        }
+        if (tdCreateTFile(&tFile, true, -1) < 0) {
+          goto _err;
+        }
 
         isFileCreated = true;
       }
@@ -1101,49 +1128,55 @@ static void *tdRSmaPersistExec(void *param) {
 
     infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash);
   }
-_normal:
+
   if (isFileCreated) {
     if (tdUpdateTFileHeader(&tFile) < 0) {
       smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile),
                tstrerror(terrno));
-      tdCloseTFile(&tFile);
-      tdRemoveTFile(&tFile);
       goto _err;
     } else {
       smaDebug("vgId:%d, rsma, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile));
     }
 
     tdCloseTFile(&tFile);
-
-    char newFName[TSDB_FILENAME_LEN];
-    strncpy(newFName, TD_TFILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN);
-    char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_F]);
-    strncpy(pos, tdQTaskInfoFname[TD_QTASK_TMP_F], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
-    if (taosRenameFile(TD_TFILE_FULL_NAME(&tFile), newFName) != 0) {
-      smaError("vgId:%d, rsma, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
-      goto _err;
-    } else {
-      smaDebug("vgId:%d, rsma, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
-    }
+    tdDestroyTFile(&tFile);
   }
-  goto _end;
+  return TSDB_CODE_SUCCESS;
 _err:
   if (isFileCreated) {
     tdRemoveTFile(&tFile);
+    tdDestroyTFile(&tFile);
   }
+  return TSDB_CODE_FAILED;
+}
+
+static void *tdRSmaPersistExec(void *param) {
+  setThreadName("rsma-task-persist");
+  SRSmaStat *pRSmaStat = param;
+  SSma      *pSma = pRSmaStat->pSma;
+
+  int8_t triggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat));
+
+  if (TASK_TRIGGER_STAT_CANCELLED == triggerStat || TASK_TRIGGER_STAT_PAUSED == triggerStat) {
+    goto _end;
+  }
+
+  // execution
+  tdRSmaPersistExecImpl(pRSmaStat);
+
 _end:
   if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
                                                                   TASK_TRIGGER_STAT_INACTIVE,
                                                                   TASK_TRIGGER_STAT_ACTIVE)) {
-    smaDebug("vgId:%d, rsma persist task is active again", vid);
+    smaDebug("vgId:%d, rsma persist task is active again", SMA_VID(pSma));
   } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
                                                                           TASK_TRIGGER_STAT_CANCELLED,
                                                                           TASK_TRIGGER_STAT_FINISHED)) {
-    smaDebug("vgId:%d, rsma persist task is cancelled", vid);
+    smaDebug("vgId:%d, rsma persist task is cancelled", SMA_VID(pSma));
   } else {
-    smaWarn("vgId:%d, rsma persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
-    ASSERT(0);
+    smaWarn("vgId:%d, rsma persist task in stat %" PRIi8, SMA_VID(pSma), atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
   }
+
   atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
   taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
   taosThreadExit(NULL);
@@ -1166,8 +1199,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
                                                                             TASK_TRIGGER_STAT_FINISHED)) {
       smaDebug("vgId:%d, persist task is cancelled and set finished", SMA_VID(pRSmaStat->pSma));
     } else {
-      smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)),
-              SMA_VID(pRSmaStat->pSma));
+      smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, SMA_VID(pRSmaStat->pSma),
+              atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
       ASSERT(0);
     }
     atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
@@ -1216,6 +1249,9 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
       atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED);
       smaDebug("rsma persistence not start since cancelled and finished");
     } break;
+    case TASK_TRIGGER_STAT_PAUSED: {
+      smaDebug("rsma persistence not start since paused");
+    } break;
     case TASK_TRIGGER_STAT_INACTIVE: {
       smaDebug("rsma persistence not start since inactive");
     } break;

source/dnode/vnode/src/sma/smaSnapshot.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sma.h"
\ No newline at end of file

source/dnode/vnode/src/sma/smaUtil.c

@@ -179,52 +179,56 @@ void tdCloseTFile(STFile *pTFile) {
   }
 }
 
-void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName) {
-  snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vgId, dname, fname);
+void tdDestroyTFile(STFile *pTFile) { taosMemoryFreeClear(TD_TFILE_FULL_NAME(pTFile)); }
+
+void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, int64_t version, char *outputName) {
+  if (version < 0) {
+    snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/v%d%s", vgId, dname, vgId, fname);
+  } else {
+    snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/v%d%s%" PRIi64, vgId, dname, vgId, fname, version);
+  }
 }
 
-int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) {
-  char    fullname[TSDB_FILENAME_LEN];
-  SDiskID did = {0};
+void tdGetVndDirName(int32_t vgId, const char *dname, char *outputName) {
+  snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", vgId, dname);
+}
 
+int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname) {
   TD_TFILE_SET_STATE(pTFile, TD_FILE_STATE_OK);
   TD_TFILE_SET_CLOSED(pTFile);
 
   memset(&(pTFile->info), 0, sizeof(pTFile->info));
   pTFile->info.magic = TD_FILE_INIT_MAGIC;
 
-  if (tfsAllocDisk(pTfs, 0, &did) < 0) {
-    terrno = TSDB_CODE_NO_AVAIL_DISK;
+  char tmpName[TSDB_FILENAME_LEN * 2 + 32] = {0};
+  snprintf(tmpName, TSDB_FILENAME_LEN * 2 + 32, "%s%s%s", dname, TD_DIRSEP, fname);
+  int32_t tmpNameLen = strlen(tmpName) + 1;
+  pTFile->fname = taosMemoryMalloc(tmpNameLen);
+  if (!pTFile->fname) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
-
-  tfsInitFile(pTfs, &(pTFile->f), did, fname);
+  tstrncpy(pTFile->fname, tmpName, tmpNameLen);
 
   return 0;
 }
 
-int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType) {
+int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType) {
   ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC);
 
   pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pTFile->pFile == NULL) {
     if (errno == ENOENT) {
       // Try to create directory recursively
-      char *s = strdup(TD_TFILE_REL_NAME(pTFile));
-      if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_TFILE_DID(pTFile)) < 0) {
-        taosMemoryFreeClear(s);
+      if (taosMulMkDir(taosDirName(TD_TFILE_FULL_NAME(pTFile))) != 0) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;
-      }
-      taosMemoryFreeClear(s);
-
+      } else {
         pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
         if (pTFile->pFile == NULL) {
           terrno = TAOS_SYSTEM_ERROR(errno);
           return -1;
         }
-    } else {
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      return -1;
       }
     }
 
@@ -240,11 +244,18 @@ int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fTyp
       tdRemoveTFile(pTFile);
       return -1;
     }
+  }
 
   return 0;
 }
 
-int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_TFILE_F(pTFile)); }
+int32_t tdRemoveTFile(STFile *pTFile) {
+  if (taosRemoveFile(TD_TFILE_FULL_NAME(pTFile)) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  };
+  return 0;
+}
 
 // smaXXXUtil ================
 // ...
\ No newline at end of file

source/dnode/vnode/src/tq/tq.c

@@ -183,13 +183,15 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
   } else {
     ASSERT(0);
   }
-  STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);
-  if (pOffset == NULL || pOffset->val.version < offset.val.version) {
+  /*STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);*/
+  /*if (pOffset != NULL) {*/
+  /*if (pOffset->val.type == TMQ_OFFSET__LOG && pOffset->val.version < offset.val.version) {*/
   if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) {
     ASSERT(0);
     return -1;
   }
-  }
+  /*}*/
+  /*}*/
 
   return 0;
 }
@@ -375,8 +377,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
     taosMemoryFree(pCkHead);
   } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    tqInfo("retrieve using snapshot req offset: uid %ld ts %ld, actual offset: uid %ld ts %ld", dataRsp.reqOffset.uid,
-           dataRsp.reqOffset.ts, fetchOffsetNew.uid, fetchOffsetNew.ts);
+    tqInfo("retrieve using snapshot actual offset: uid %ld ts %ld", fetchOffsetNew.uid, fetchOffsetNew.ts);
     if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) {
       ASSERT(0);
     }

source/dnode/vnode/src/tq/tqRead.c

@@ -120,7 +120,9 @@ bool tqNextDataBlock(SStreamReader* pHandle) {
       return true;
     }
     void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->msgIter.uid, sizeof(int64_t));
+    /*tqDebug("search uid %ld", pHandle->msgIter.uid);*/
     if (ret != NULL) {
+      /*tqDebug("find   uid %ld", pHandle->msgIter.uid);*/
       return true;
     }
   }

source/dnode/vnode/src/tq/tqSink.c

@@ -185,5 +185,7 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
       .contLen = ntohl(pReq->length),
   };
 
-  ASSERT(tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) == 0);
+  if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
+    tqDebug("failed to put into write-queue since %s", terrstr());
+  }
 }

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -28,12 +28,12 @@ const SVnodeCfg vnodeCfgDefault = {
                 .update = 1,
                 .compression = 2,
                 .slLevel = 5,
-                .days = 10,
+                .days = 14400,
                 .minRows = 100,
                 .maxRows = 4096,
-                .keep2 = 3650,
-                .keep0 = 3650,
-                .keep1 = 3650},
+                .keep2 = 5256000,
+                .keep0 = 5256000,
+                .keep1 = 5256000},
     .walCfg =
         {.vgId = -1, .fsyncPeriod = 0, .retentionPeriod = 0, .rollPeriod = 0, .segSize = 0, .level = TAOS_WAL_WRITE},
     .hashBegin = 0,

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -229,6 +229,9 @@ int vnodeCommit(SVnode *pVnode) {
     return -1;
   }
 
+  // preCommit
+  // TODO
+  
   // commit each sub-system
   if (metaCommit(pVnode->pMeta) < 0) {
     ASSERT(0);
@@ -269,6 +272,9 @@ int vnodeCommit(SVnode *pVnode) {
   
   pVnode->state.committed = info.state.committed;
 
+  // postCommit
+  smaPostCommit(pVnode->pSma);
+
   // apply the commit (TODO)
   vnodeBufPoolReset(pVnode->onCommit);
   pVnode->onCommit->next = pVnode->pPool;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -779,6 +779,7 @@ _exit:
   taosArrayDestroy(submitRsp.pArray);
 
   // TODO: the partial success scenario and the error case
+  // => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level 1/level 2.
   // TODO: refactor
   if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
     tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);

source/libs/catalog/inc/catalogInt.h

@@ -278,7 +278,7 @@ typedef struct SCtgAsyncFps {
 
 typedef struct SCtgApiStat {
 
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 

source/libs/catalog/test/CMakeLists.txt

 
 MESSAGE(STATUS "build catalog unit test")
 
-# GoogleTest requires at least C++11
-SET(CMAKE_CXX_STANDARD 11)
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+IF(NOT TD_DARWIN)
+        # GoogleTest requires at least C++11
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
-TARGET_LINK_LIBRARIES(
+        ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
+        TARGET_LINK_LIBRARIES(
                 catalogTest
                 PUBLIC os util common catalog transport gtest qcom taos_static
-)
+        )
 
-TARGET_INCLUDE_DIRECTORIES(
+        TARGET_INCLUDE_DIRECTORIES(
                 catalogTest
                 PUBLIC "${TD_SOURCE_DIR}/include/libs/catalog/"
                 PRIVATE "${TD_SOURCE_DIR}/source/libs/catalog/inc"
-)
+        )
 
-# add_test(
-#     NAME catalogTest
-#     COMMAND catalogTest
-# )
+        # add_test(
+        #     NAME catalogTest
+        #     COMMAND catalogTest
+        # )
+ENDIF()

source/libs/executor/inc/executorimpl.h

@@ -356,6 +356,7 @@ typedef struct SStreamBlockScanInfo {
   SUpdateInfo*    pUpdateInfo;
 
   EStreamScanMode scanMode;
+  SOperatorInfo* pStreamScanOp;
   SOperatorInfo* pSnapshotReadOp;
   SArray*        childIds;
   SessionWindowSupporter sessionSup;
@@ -427,7 +428,7 @@ typedef struct SIntervalAggOperatorInfo {
   STimeWindowAggSupp twAggSup;
   bool               invertible;
   SArray*            pPrevValues;        //  SArray<SGroupKeys> used to keep the previous not null value for interpolation.
-  bool               ignoreCloseWindow;
+  bool               ignoreExpiredData;
 } SIntervalAggOperatorInfo;
 
 typedef struct SStreamFinalIntervalOperatorInfo {
@@ -449,7 +450,7 @@ typedef struct SStreamFinalIntervalOperatorInfo {
   SArray*            pPullWins;          // SPullWindowInfo
   int32_t            pullIndex;
   SSDataBlock*       pPullDataRes;
-  bool               ignoreCloseWindow;
+  bool               ignoreExpiredData;
 } SStreamFinalIntervalOperatorInfo;
 
 typedef struct SAggOperatorInfo {
@@ -587,7 +588,7 @@ typedef struct SStreamSessionAggOperatorInfo {
   SArray*              pChildren;       // cache for children's result; final stream operator
   SPhysiNode*          pPhyNode;           // create new child
   bool                 isFinal;
-  bool                 ignoreCloseWindow;
+  bool                 ignoreExpiredData;
 } SStreamSessionAggOperatorInfo;
 
 typedef struct STimeSliceOperatorInfo {
@@ -631,7 +632,7 @@ typedef struct SStreamStateAggOperatorInfo {
   void*                pDelIterator;
   SArray*              pScanWindow;
   SArray*              pChildren;       // cache for children's result;
-  bool                 ignoreCloseWindow;
+  bool                 ignoreExpiredData;
 } SStreamStateAggOperatorInfo;
 
 typedef struct SSortedMergeOperatorInfo {

source/libs/executor/src/dataSinkMgt.c

@@ -35,7 +35,7 @@ int32_t dsDataSinkGetCacheSize(SDataSinkStat *pStat) {
 
 
 int32_t dsCreateDataSinker(const SDataSinkNode *pDataSink, DataSinkHandle* pHandle, void* pParam) {
-  switch (nodeType(pDataSink)) {
+  switch ((int)nodeType(pDataSink)) {
     case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
       return createDataDispatcher(&gDataSinkManager, pDataSink, pHandle);
     case QUERY_NODE_PHYSICAL_PLAN_DELETE:

source/libs/executor/src/executor.c

@@ -145,10 +145,12 @@ static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo,
       continue;
     }
 
+    // TODO handle ntb case
     if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) {
       continue;
     }
-    // TODO handle ntb case
+    /*pScanInfo->pStreamScanOp->pTaskInfo->tableqinfoList.*/
+    // handle multiple partition
 
     taosArrayPush(qa, id);
   }

source/libs/executor/src/executorimpl.c

@@ -2027,8 +2027,9 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
 
   ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY);
 
-  qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, GET_TASKID(pTaskInfo),
-         pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, sourceIndex, totalSources);
+  qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu,
+         GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId,
+         sourceIndex, totalSources);
 
   pMsg->header.vgId = htonl(pSource->addr.nodeId);
   pMsg->sId = htobe64(pSource->schedId);
@@ -2163,8 +2164,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
       SSDataBlock*         pRes = pExchangeInfo->pResult;
       SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
       if (pRsp->numOfRows == 0) {
-        qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
-               ", completed:%d try next %d/%" PRIzu,
+        qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64
+               ", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu,
                GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows,
                pExchangeInfo->loadInfo.totalRows, completed + 1, i + 1, totalSources);
         pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
@@ -2183,18 +2184,19 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
       }
 
       if (pRsp->completed == 1) {
-        qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d"
+        qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64
+               " execId:%d"
                " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64
                ", completed:%d try next %d/%" PRIzu,
-               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows, pDataInfo->totalRows,
-               pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources);
+               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows,
+               pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources);
         completed += 1;
         pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
       } else {
         qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
                ", totalBytes:%" PRIu64,
-               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows,
-               pLoadInfo->totalSize);
+               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
+               pLoadInfo->totalRows, pLoadInfo->totalSize);
       }
 
       taosMemoryFreeClear(pDataInfo->pRsp);
@@ -2267,8 +2269,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
 
     if (pDataInfo->code != TSDB_CODE_SUCCESS) {
-      qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), pSource->addr.nodeId,
-             pSource->taskId, pSource->execId, tstrerror(pDataInfo->code));
+      qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo),
+             pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code));
       pOperator->pTaskInfo->code = pDataInfo->code;
       return NULL;
     }
@@ -2276,8 +2278,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     SRetrieveTableRsp*   pRsp = pDataInfo->pRsp;
     SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
     if (pRsp->numOfRows == 0) {
-      qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
-             " try next",
+      qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64
+             ", totalRows:%" PRIu64 " try next",
              GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1,
              pDataInfo->totalRows, pLoadInfo->totalRows);
 
@@ -2296,16 +2298,17 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     if (pRsp->completed == 1) {
       qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64
              ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
-             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pDataInfo->totalRows,
-             pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources);
+             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
+             pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1,
+             totalSources);
 
       pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
       pExchangeInfo->current += 1;
     } else {
       qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
              ", totalBytes:%" PRIu64,
-             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows,
-             pLoadInfo->totalSize);
+             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
+             pLoadInfo->totalRows, pLoadInfo->totalSize);
     }
 
     pOperator->resultInfo.totalRows += pRes->info.rows;
@@ -3250,6 +3253,10 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       doSetOperatorCompleted(pOperator);
       break;
     }
+    if (pBlock->info.type == STREAM_RETRIEVE) {
+      // for stream interval
+      return pBlock;
+    }
 
     // the pDataBlock are always the same one, no need to call this again
     int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);
@@ -4094,7 +4101,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
       ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
       SValueNode* pValue = (SValueNode*)pNew;
 
-      if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) {
+      if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL || pValue->isNull) {
         isNull[index++] = 1;
         continue;
       } else {

source/libs/executor/src/scanoperator.c

@@ -807,6 +807,23 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
   return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
 }
 
+static void setGroupId(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
+  ASSERT(rowIndex < pBlock->info.rows);
+  switch (pBlock->info.type)
+  {
+  case STREAM_RETRIEVE: {
+    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
+    uint64_t* groupCol = (uint64_t*)pColInfo->pData;
+    pInfo->groupId = groupCol[rowIndex];
+  }
+    break;
+  case STREAM_DELETE_DATA:
+    break;
+  default:
+    break;
+  }
+}
+
 static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
   STimeWindow win = {
       .skey = INT64_MIN,
@@ -829,6 +846,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3
     } else {
       win =
           getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL);
+      setGroupId(pInfo, pSDB, 2, *pRowIndex);
       (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL,
                                                TSDB_ORDER_ASC);
     }
@@ -1031,10 +1049,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
       }
       pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
     } else {
-      if (isStateWindow(pInfo) && taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) {
+      if (isStateWindow(pInfo)) {
         pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
         pInfo->updateResIndex = pInfo->pUpdateRes->info.rows;
-        prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
+        if (!prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex)) {
+          pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
+        }
       }
       if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) {
         SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
@@ -1274,6 +1294,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
   pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1};
   pInfo->groupId = 0;
   pInfo->pPullDataRes = createPullDataBlock();
+  pInfo->pStreamScanOp = pOperator;
 
   pOperator->name = "StreamBlockScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;

source/libs/executor/src/timewindowoperator.c

@@ -838,7 +838,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
   STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
                                         pInfo->interval.precision, &pInfo->win);
   int32_t     ret = TSDB_CODE_SUCCESS;
-  if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
+  if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) {
     ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                  numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
     if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
@@ -871,7 +871,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
     doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
   }
 
-  if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
+  if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) {
     updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
     doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
                      pBlock->info.rows, numOfOutput, pInfo->order);
@@ -886,7 +886,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
     if (startPos < 0) {
       break;
     }
-    if (pInfo->ignoreCloseWindow && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
+    if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
       ekey = ascScan ? nextWin.ekey : nextWin.skey;
       forwardRows =
           getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
@@ -1535,7 +1535,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
   pInfo->interval = *pInterval;
   pInfo->execModel = pTaskInfo->execModel;
   pInfo->twAggSup = *pTwAggSupp;
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pPhyNode->window.igExpired;
 
   if (pPhyNode->window.pExprs != NULL) {
     int32_t    numOfScalar = 0;
@@ -2292,7 +2292,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
                                             pInfo->interval.precision, NULL);
   while (1) {
     bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
-    if (pInfo->ignoreCloseWindow && isClosed) {
+    if (pInfo->ignoreExpiredData && isClosed) {
       startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
       if (startPos < 0) {
         break;
@@ -2710,7 +2710,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
   pInfo->pPullDataRes = createPullDataBlock();
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
 
   pOperator->operatorType = pPhyNode->type;
   pOperator->blocking = true;
@@ -2852,12 +2852,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
   pInfo->pDelIterator = NULL;
   pInfo->pDelRes = createOneDataBlock(pResBlock, false);
-  pInfo->pDelRes->info.type = STREAM_DELETE;
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
   blockDataEnsureCapacity(pInfo->pDelRes, 64);
   pInfo->pChildren = NULL;
   pInfo->isFinal = false;
   pInfo->pPhyNode = pPhyNode;
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
 
   pOperator->name = "StreamSessionWindowAggOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
@@ -3133,7 +3133,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
 
   SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
   for (int32_t i = 0; i < pSDataBlock->info.rows;) {
-    if (pInfo->ignoreCloseWindow && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
+    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
       i++;
       continue;
     }
@@ -3413,8 +3413,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
   pOperator->status = OP_RES_TO_RETURN;
 
   closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
-      getResWinForSession, pInfo->ignoreCloseWindow);
-  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
+      getResWinForSession, pInfo->ignoreExpiredData);
+  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
   copyUpdateResult(pStUpdated, pUpdated);
   taosHashCleanup(pStUpdated);
 
@@ -3822,7 +3822,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
   SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
   SColumnInfoData*     pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
   for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
-    if (pInfo->ignoreCloseWindow && isOverdue(tsCols[i], &pInfo->twAggSup)) {
+    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
       i++;
       continue;
     }
@@ -3866,12 +3866,14 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_RES_TO_RETURN) {
     doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
     if (pInfo->pDelRes->info.rows > 0) {
+      printDataBlock(pInfo->pDelRes, "single state");
       return pInfo->pDelRes;
     }
     doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
     if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
     }
+    printDataBlock(pBInfo->pRes, "single state");
     return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
   }
 
@@ -3884,6 +3886,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
     if (pBlock == NULL) {
       break;
     }
+    printDataBlock(pBlock, "single state recv");
 
     if (pBlock->info.type == STREAM_CLEAR) {
       doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
@@ -3903,8 +3906,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   pOperator->status = OP_RES_TO_RETURN;
 
   closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
-      getResWinForState, pInfo->ignoreCloseWindow);
-  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
+      getResWinForState, pInfo->ignoreExpiredData);
+  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
   copyUpdateResult(pSeUpdated, pUpdated);
   taosHashCleanup(pSeUpdated);
 
@@ -3914,9 +3917,11 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
   doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
   if (pInfo->pDelRes->info.rows > 0) {
+    printDataBlock(pInfo->pDelRes, "single state");
     return pInfo->pDelRes;
   }
   doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
+  printDataBlock(pBInfo->pRes, "single state");
   return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
 }
 
@@ -3975,10 +3980,10 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
   pInfo->pDelIterator = NULL;
   pInfo->pDelRes = createOneDataBlock(pResBlock, false);
-  pInfo->pDelRes->info.type = STREAM_DELETE;
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
   blockDataEnsureCapacity(pInfo->pDelRes, 64);
   pInfo->pChildren = NULL;
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pStateNode->window.igExpired;
 
   pOperator->name = "StreamStateAggOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;

source/libs/function/src/builtins.c

@@ -39,6 +39,174 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char
   return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName);
 }
 
+#define TIME_UNIT_INVALID 1
+#define TIME_UNIT_TOO_SMALL 2
+
+static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
+  if (!pVal->isDuration) {
+    return TIME_UNIT_INVALID;
+  }
+
+  if (TSDB_TIME_PRECISION_MILLI == dbPrec && 0 == strcasecmp(pVal->literal, "1u")) {
+    return TIME_UNIT_TOO_SMALL;
+  }
+
+  if (pVal->literal[0] != '1' || (pVal->literal[1] != 'u' && pVal->literal[1] != 'a' &&
+                                  pVal->literal[1] != 's' && pVal->literal[1] != 'm' &&
+                                  pVal->literal[1] != 'h' && pVal->literal[1] != 'd' &&
+                                  pVal->literal[1] != 'w')) {
+    return TIME_UNIT_INVALID;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/* Following are valid ISO-8601 timezone format:
+ * 1 z/Z
+ * 2 ±hh:mm
+ * 3 ±hhmm
+ * 4 ±hh
+ *
+ */
+
+static bool validateHourRange(int8_t hour) {
+  if (hour < 0 || hour > 12) {
+    return false;
+  }
+
+  return true;
+}
+
+static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
+  if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '+')) {
+    return true;
+  }
+
+  return false;
+}
+
+static bool validateTimestampDigits(const SValueNode* pVal) {
+  if (!IS_INTEGER_TYPE(pVal->node.resType.type)) {
+    return false;
+  }
+
+  int64_t tsVal = pVal->datum.i;
+  char    fraction[20] = {0};
+  NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction);
+  int32_t tsDigits = (int32_t)strlen(fraction);
+
+  if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
+    if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
+        tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+static bool validateTimezoneFormat(const SValueNode* pVal) {
+  if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
+    return false;
+  }
+
+  char*   tz = varDataVal(pVal->datum.p);
+  int32_t len = varDataLen(pVal->datum.p);
+
+  char   buf[3] = {0};
+  int8_t hour = -1, minute = -1;
+  if (len == 0) {
+    return false;
+  } else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) {
+    return true;
+  } else if ((tz[0] == '+' || tz[0] == '-')) {
+    switch (len) {
+      case 3:
+      case 5: {
+        for (int32_t i = 1; i < len; ++i) {
+          if (!isdigit(tz[i])) {
+            return false;
+          }
+
+          if (i == 2) {
+            memcpy(buf, &tz[i - 1], 2);
+            hour = taosStr2Int8(buf, NULL, 10);
+            if (!validateHourRange(hour)) {
+              return false;
+            }
+          } else if (i == 4) {
+            memcpy(buf, &tz[i - 1], 2);
+            minute = taosStr2Int8(buf, NULL, 10);
+            if (!validateMinuteRange(hour, minute, tz[0])) {
+              return false;
+            }
+          }
+        }
+        break;
+      }
+      case 6: {
+        for (int32_t i = 1; i < len; ++i) {
+          if (i == 3) {
+            if (tz[i] != ':') {
+              return false;
+            }
+            continue;
+          }
+
+          if (!isdigit(tz[i])) {
+            return false;
+          }
+
+          if (i == 2) {
+            memcpy(buf, &tz[i - 1], 2);
+            hour = taosStr2Int8(buf, NULL, 10);
+            if (!validateHourRange(hour)) {
+              return false;
+            }
+          } else if (i == 5) {
+            memcpy(buf, &tz[i - 1], 2);
+            minute = taosStr2Int8(buf, NULL, 10);
+            if (!validateMinuteRange(hour, minute, tz[0])) {
+              return false;
+            }
+          }
+        }
+        break;
+      }
+      default: {
+        return false;
+      }
+    }
+  } else {
+    return false;
+  }
+
+  return true;
+}
+
+void static addTimezoneParam(SNodeList* pList) {
+  char       buf[6] = {0};
+  time_t     t = taosTime(NULL);
+  struct tm* tmInfo = taosLocalTime(&t, NULL);
+  strftime(buf, sizeof(buf), "%z", tmInfo);
+  int32_t len = (int32_t)strlen(buf);
+
+  SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
+  pVal->literal = strndup(buf, len);
+  pVal->isDuration = false;
+  pVal->translate = true;
+  pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
+  pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
+  pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
+  pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1);
+  varDataSetLen(pVal->datum.p, len);
+  strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
+
+  nodesListAppend(pList, (SNode*)pVal);
+}
+
 void static addDbPrecisonParam(SNodeList** pList, uint8_t precision) {
   SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
   pVal->literal = NULL;
@@ -525,9 +693,15 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len
       return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
     }
 
-    if (pValue->datum.i == 0) {
+    uint8_t dbPrec = pFunc->node.resType.precision;
+
+    int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1));
+    if (ret == TIME_UNIT_TOO_SMALL) {
       return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                              "ELAPSED function time unit parameter should be greater than db precision");
+    } else if (ret == TIME_UNIT_INVALID) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "ELAPSED function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
     }
   }
 
@@ -843,6 +1017,19 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
+  if (numOfParams == 4) {
+    uint8_t dbPrec = pFunc->node.resType.precision;
+
+    int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 3));
+    if (ret == TIME_UNIT_TOO_SMALL) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "STATEDURATION function time unit parameter should be greater than db precision");
+    } else if (ret == TIME_UNIT_INVALID) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "STATEDURATION function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+    }
+  }
+
   // set result type
   pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
   return TSDB_CODE_SUCCESS;
@@ -1259,19 +1446,9 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
 
 static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // The number of parameters has been limited by the syntax definition
-  // uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
 
   // The function return type has been set during syntax parsing
   uint8_t para2Type = pFunc->node.resType.type;
-  // if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
-  //     para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
-  //     para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
-  //   return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
-  // }
-  // if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
-  //     (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
-  //   return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
-  // }
 
   int32_t para2Bytes = pFunc->node.resType.bytes;
   if (IS_VAR_DATA_TYPE(para2Type)) {
@@ -1281,153 +1458,12 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
     return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                            "CAST function converted length should be in range [0, 1000]");
   }
-  return TSDB_CODE_SUCCESS;
-}
-
-/* Following are valid ISO-8601 timezone format:
- * 1 z/Z
- * 2 ±hh:mm
- * 3 ±hhmm
- * 4 ±hh
- *
- */
-
-static bool validateHourRange(int8_t hour) {
-  if (hour < 0 || hour > 12) {
-    return false;
-  }
-
-  return true;
-}
-
-static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
-  if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '+')) {
-    return true;
-  }
-
-  return false;
-}
-
-static bool validateTimestampDigits(const SValueNode* pVal) {
-  if (!IS_INTEGER_TYPE(pVal->node.resType.type)) {
-    return false;
-  }
-
-  int64_t tsVal = pVal->datum.i;
-  char    fraction[20] = {0};
-  NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction);
-  int32_t tsDigits = (int32_t)strlen(fraction);
-
-  if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
-    if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
-        tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
-      return true;
-    } else {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-static bool validateTimezoneFormat(const SValueNode* pVal) {
-  if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
-    return false;
-  }
-
-  char*   tz = varDataVal(pVal->datum.p);
-  int32_t len = varDataLen(pVal->datum.p);
-
-  char   buf[3] = {0};
-  int8_t hour = -1, minute = -1;
-  if (len == 0) {
-    return false;
-  } else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) {
-    return true;
-  } else if ((tz[0] == '+' || tz[0] == '-')) {
-    switch (len) {
-      case 3:
-      case 5: {
-        for (int32_t i = 1; i < len; ++i) {
-          if (!isdigit(tz[i])) {
-            return false;
-          }
-
-          if (i == 2) {
-            memcpy(buf, &tz[i - 1], 2);
-            hour = taosStr2Int8(buf, NULL, 10);
-            if (!validateHourRange(hour)) {
-              return false;
-            }
-          } else if (i == 4) {
-            memcpy(buf, &tz[i - 1], 2);
-            minute = taosStr2Int8(buf, NULL, 10);
-            if (!validateMinuteRange(hour, minute, tz[0])) {
-              return false;
-            }
-          }
-        }
-        break;
-      }
-      case 6: {
-        for (int32_t i = 1; i < len; ++i) {
-          if (i == 3) {
-            if (tz[i] != ':') {
-              return false;
-            }
-            continue;
-          }
-
-          if (!isdigit(tz[i])) {
-            return false;
-          }
-
-          if (i == 2) {
-            memcpy(buf, &tz[i - 1], 2);
-            hour = taosStr2Int8(buf, NULL, 10);
-            if (!validateHourRange(hour)) {
-              return false;
-            }
-          } else if (i == 5) {
-            memcpy(buf, &tz[i - 1], 2);
-            minute = taosStr2Int8(buf, NULL, 10);
-            if (!validateMinuteRange(hour, minute, tz[0])) {
-              return false;
-            }
-          }
-        }
-        break;
-      }
-      default: {
-        return false;
-      }
-    }
-  } else {
-    return false;
-  }
 
-  return true;
-}
-
-void static addTimezoneParam(SNodeList* pList) {
-  char       buf[6] = {0};
-  time_t     t = taosTime(NULL);
-  struct tm* tmInfo = taosLocalTime(&t, NULL);
-  strftime(buf, sizeof(buf), "%z", tmInfo);
-  int32_t len = (int32_t)strlen(buf);
-
-  SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
-  pVal->literal = strndup(buf, len);
-  pVal->isDuration = false;
-  pVal->translate = true;
-  pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
-  pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
-  pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
-  pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1);
-  varDataSetLen(pVal->datum.p, len);
-  strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
+  // add database precision as param
+  uint8_t dbPrec = pFunc->node.resType.precision;
+  addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
 
-  nodesListAppend(pList, (SNode*)pVal);
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
@@ -1498,6 +1534,16 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
 
   // add database precision as param
   uint8_t dbPrec = pFunc->node.resType.precision;
+
+  int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1));
+  if (ret == TIME_UNIT_TOO_SMALL) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "TIMETRUNCATE function time unit parameter should be greater than db precision");
+  } else if (ret == TIME_UNIT_INVALID) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "TIMETRUNCATE function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+  }
+
   addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
 
   pFunc->node.resType =
@@ -1526,6 +1572,18 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
 
   // add database precision as param
   uint8_t dbPrec = pFunc->node.resType.precision;
+
+  if (3 == numOfParams) {
+    int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 2));
+    if (ret == TIME_UNIT_TOO_SMALL) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "TIMEDIFF function time unit parameter should be greater than db precision");
+    } else if (ret == TIME_UNIT_INVALID) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "TIMEDIFF function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+    }
+  }
+
   addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
 
   pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};

source/libs/index/CMakeLists.txt

@@ -35,7 +35,7 @@ if (${BUILD_WITH_INVERTEDINDEX})
 endif(${BUILD_WITH_INVERTEDINDEX})
 
 
-if (${BUILD_TEST})
-  add_subdirectory(test)
-endif(${BUILD_TEST})
+# if (${BUILD_TEST})
+#   add_subdirectory(test)
+# endif(${BUILD_TEST})
 

source/libs/index/src/indexFstSparse.c

@@ -17,7 +17,7 @@
 
 FstSparseSet *sparSetCreate(int32_t sz) {
   FstSparseSet *ss = taosMemoryCalloc(1, sizeof(FstSparseSet));
-  if (ss = NULL) {
+  if (ss == NULL) {
     return NULL;
   }
 

source/libs/monitor/test/monTest.cpp

@@ -75,18 +75,18 @@ void MonitorTest::GetSysInfo(SMonSysInfo *pInfo) {
   pInfo->cpu_engine = 2.1;
   pInfo->cpu_system = 2.1;
   pInfo->cpu_cores = 2;
-  pInfo->mem_engine = 3.1;
-  pInfo->mem_system = 3.2;
-  pInfo->mem_total = 3.3;
-  pInfo->disk_engine = 4.1;
-  pInfo->disk_used = 4.2;
-  pInfo->disk_total = 4.3;
-  pInfo->net_in = 5.1;
-  pInfo->net_out = 5.2;
-  pInfo->io_read = 6.1;
-  pInfo->io_write = 6.2;
-  pInfo->io_read_disk = 7.1;
-  pInfo->io_write_disk = 7.2;
+  pInfo->mem_engine = 3;
+  pInfo->mem_system = 3;
+  pInfo->mem_total = 3;
+  pInfo->disk_engine = 4;
+  pInfo->disk_used = 4;
+  pInfo->disk_total = 4;
+  pInfo->net_in = 5;
+  pInfo->net_out = 5;
+  pInfo->io_read = 6;
+  pInfo->io_write = 6;
+  pInfo->io_read_disk = 7;
+  pInfo->io_write_disk = 7;
 }
 
 void MonitorTest::GetClusterInfo(SMonClusterInfo *pInfo) {

source/libs/parser/test/CMakeLists.txt

 
 MESSAGE(STATUS "build parser unit test")
 
-# GoogleTest requires at least C++11
-SET(CMAKE_CXX_STANDARD 11)
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+IF(NOT TD_DARWIN)
+    # GoogleTest requires at least C++11
+    SET(CMAKE_CXX_STANDARD 11)
+    AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(parserTest ${SOURCE_LIST})
+    ADD_EXECUTABLE(parserTest ${SOURCE_LIST})
 
-TARGET_INCLUDE_DIRECTORIES(
+    TARGET_INCLUDE_DIRECTORIES(
             parserTest
             PUBLIC "${TD_SOURCE_DIR}/include/libs/parser/"
             PRIVATE "${TD_SOURCE_DIR}/source/libs/parser/inc"
-)
+    )
 
-TARGET_LINK_LIBRARIES(
+    TARGET_LINK_LIBRARIES(
             parserTest
             PUBLIC os util common nodes parser catalog transport gtest function planner qcom
-)
+    )
 
-if(${BUILD_WINGETOPT})
+    if(${BUILD_WINGETOPT})
         target_include_directories(
             parserTest
             PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
         )
         target_link_libraries(parserTest PUBLIC wingetopt)
-endif()
+    endif()
 
-add_test(
+    add_test(
         NAME parserTest
         COMMAND parserTest
-)
+    )
+ENDIF()
\ No newline at end of file

source/libs/qworker/test/CMakeLists.txt

 
 MESSAGE(STATUS "build qworker unit test")
+IF(NOT TD_DARWIN)
+        # GoogleTest requires at least C++11
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-# GoogleTest requires at least C++11
-SET(CMAKE_CXX_STANDARD 11)
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
-
-ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST})
-TARGET_LINK_LIBRARIES(
+        ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST})
+        TARGET_LINK_LIBRARIES(
                 qworkerTest
                 PUBLIC os util common transport gtest qcom nodes planner qworker executor
-)
+        )
 
-TARGET_INCLUDE_DIRECTORIES(
+        TARGET_INCLUDE_DIRECTORIES(
                 qworkerTest
                 PUBLIC "${TD_SOURCE_DIR}/include/libs/qworker/"
                 PRIVATE "${TD_SOURCE_DIR}/source/libs/qworker/inc"
-)
+        )
+ENDIF()

source/libs/scalar/src/filter.c

@@ -196,7 +196,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
         terrno = TSDB_CODE_QRY_JSON_IN_ERROR;
         return 0;
       default:
-        assert(0);
+        return 0;
     }
   }
 
@@ -222,7 +222,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
         terrno = TSDB_CODE_QRY_JSON_IN_ERROR;
         return 0;
       default:
-        assert(0);
+        return 0;
     }
   }
 

source/libs/scalar/src/scalar.c

@@ -109,9 +109,8 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
       }
 
       if (IS_VAR_DATA_TYPE(type)) {
-        char* data = colDataGetVarData(out.columnData, 0);
-        len = varDataLen(data);
-        buf = varDataVal(data);
+        buf = colDataGetVarData(out.columnData, 0);
+        len = varDataTLen(data);
       } else {
         len = tDataTypes[type].bytes;
         buf = out.columnData->pData;
@@ -119,8 +118,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
     } else {
       buf = nodesGetValueFromNode(valueNode);
       if (IS_VAR_DATA_TYPE(type)) {
-        len = varDataLen(buf);
-        buf = varDataVal(buf);
+        len = varDataTLen(buf);
       } else {
         len = valueNode->node.resType.bytes;
       }
@@ -194,7 +192,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
 
       param->numOfRows = 1;
       param->columnData = sclCreateColumnInfoData(&valueNode->node.resType, 1);
-      if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) {
+      if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) {
         colDataAppendNULL(param->columnData, 0);
       } else {
         colDataAppend(param->columnData, 0, nodesGetValueFromNode(valueNode), false);
@@ -345,7 +343,7 @@ int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx) {
     return -1;
   }
   
-  switch (nodeType(pNode)) {
+  switch ((int)nodeType(pNode)) {
     case QUERY_NODE_VALUE: {
       SValueNode *valueNode = (SValueNode *)pNode;
       return valueNode->node.resType.type;
@@ -538,6 +536,14 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
   int32_t rowNum = 0;
   int32_t code = 0;
 
+  // json not support in in operator
+  if(nodeType(node->pLeft) == QUERY_NODE_VALUE){
+    SValueNode *valueNode = (SValueNode *)node->pLeft;
+    if(valueNode->node.resType.type == TSDB_DATA_TYPE_JSON && (node->opType == OP_TYPE_IN || node->opType == OP_TYPE_NOT_IN)){
+      SCL_RET(TSDB_CODE_QRY_JSON_IN_ERROR);
+    }
+  }
+
   SCL_ERR_RET(sclInitOperatorParams(&params, node, ctx, &rowNum));
   output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum);
   if (output->columnData == NULL) {
@@ -777,7 +783,12 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) {
   res->translate = true;
 
   if (colDataIsNull_s(output.columnData, 0)) {
+    if(node->node.resType.type != TSDB_DATA_TYPE_JSON){
       res->node.resType.type = TSDB_DATA_TYPE_NULL;
+    }else{
+      res->node.resType = node->node.resType;
+      res->isNull = true;
+    }
   } else {
     res->node.resType = node->node.resType;
     int32_t type = output.columnData->info.type;

source/libs/scalar/src/sclfunc.c

@@ -934,9 +934,16 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
         break;
       }
       case TSDB_DATA_TYPE_TIMESTAMP: {
+        int64_t timeVal;
         if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) {
-          //convert to 0
+          int64_t timePrec;
+          GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
+          int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal);
+          if (ret != TSDB_CODE_SUCCESS) {
             *(int64_t *)output = 0;
+          } else {
+            *(int64_t *)output = timeVal;
+          }
         } else {
           GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input);
         }

source/libs/scalar/test/filter/CMakeLists.txt

 
 MESSAGE(STATUS "build filter unit test")
 
-# GoogleTest requires at least C++11
-SET(CMAKE_CXX_STANDARD 11)
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+IF(NOT TD_DARWIN)
+        # GoogleTest requires at least C++11
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(filterTest ${SOURCE_LIST})
-TARGET_LINK_LIBRARIES(
+        ADD_EXECUTABLE(filterTest ${SOURCE_LIST})
+        TARGET_LINK_LIBRARIES(
                 filterTest
                 PUBLIC os util common gtest qcom function nodes scalar
-)
+        )
 
-TARGET_INCLUDE_DIRECTORIES(
+        TARGET_INCLUDE_DIRECTORIES(
                 filterTest
                 PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/"
                 PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
-)
+        )
+ENDIF()
\ No newline at end of file

source/libs/scalar/test/scalar/CMakeLists.txt

 
 MESSAGE(STATUS "build scalar unit test")
 
-# GoogleTest requires at least C++11
-SET(CMAKE_CXX_STANDARD 11)
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+IF(NOT TD_DARWIN)
+        # GoogleTest requires at least C++11
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(scalarTest ${SOURCE_LIST})
-TARGET_LINK_LIBRARIES(
+        ADD_EXECUTABLE(scalarTest ${SOURCE_LIST})
+        TARGET_LINK_LIBRARIES(
                 scalarTest
                 PUBLIC os util common gtest qcom function nodes scalar parser catalog transport
-)
+        )
 
-TARGET_INCLUDE_DIRECTORIES(
+        TARGET_INCLUDE_DIRECTORIES(
                 scalarTest
                 PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/"
                 PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc"
                 PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
-)
-add_test(
+        )
+        add_test(
                 NAME scalarTest
                 COMMAND scalarTest
-)
+        )
+ENDIF()

source/libs/scheduler/inc/schedulerInt.h

@@ -68,7 +68,7 @@ typedef struct SSchHbTrans {
 
 typedef struct SSchApiStat {
 
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 
@@ -76,7 +76,7 @@ typedef struct SSchApiStat {
 
 typedef struct SSchRuntimeStat {
 
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 
@@ -84,7 +84,7 @@ typedef struct SSchRuntimeStat {
 
 typedef struct SSchJobStat {
 
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 

source/libs/scheduler/test/CMakeLists.txt

 
 MESSAGE(STATUS "build scheduler unit test")
 
-# GoogleTest requires at least C++11
-SET(CMAKE_CXX_STANDARD 11)
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+IF(NOT TD_DARWIN)
+        # GoogleTest requires at least C++11
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(schedulerTest ${SOURCE_LIST})
-TARGET_LINK_LIBRARIES(
+        ADD_EXECUTABLE(schedulerTest ${SOURCE_LIST})
+        TARGET_LINK_LIBRARIES(
                 schedulerTest
                 PUBLIC os util common catalog transport gtest qcom taos_static planner scheduler
-)
+        )
 
-TARGET_INCLUDE_DIRECTORIES(
+        TARGET_INCLUDE_DIRECTORIES(
                 schedulerTest
                 PUBLIC "${TD_SOURCE_DIR}/include/libs/scheduler/"
                 PRIVATE "${TD_SOURCE_DIR}/source/libs/scheduler/inc"
-)
+        )
+ENDIF()
\ No newline at end of file

source/libs/tdb/src/db/tdbTxn.c

@@ -18,7 +18,7 @@
 int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg,
                int flags) {
   // not support read-committed version at the moment
-  ASSERT(flags == 0 || flags == TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+  ASSERT(flags == 0 || flags == (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
 
   pTxn->flags = flags;
   pTxn->txnId = txnid;

source/os/CMakeLists.txt

@@ -39,12 +39,16 @@ endif()
 target_link_libraries(
     os PUBLIC pthread
 )
-if(NOT TD_WINDOWS)
+if(TD_WINDOWS)
     target_link_libraries(
-        os PUBLIC dl m rt
+        os PUBLIC ws2_32 iconv msvcregex wcwidth winmm
+    )
+elseif(TD_DARWIN_64)
+    target_link_libraries(
+        os PUBLIC dl m iconv
     )
 else()
     target_link_libraries(
-        os PUBLIC ws2_32 iconv msvcregex wcwidth winmm
+        os PUBLIC dl m rt
     )
-endif(NOT TD_WINDOWS)
+endif()

source/os/src/osAtomic.c

@@ -518,7 +518,7 @@ int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) {
 #endif
 }
 
-void* atomic_add_fetch_ptr(void *ptr, void *val) {
+void* atomic_add_fetch_ptr(void *ptr, int64_t val) {
 #ifdef WINDOWS
   return interlocked_add_fetch_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
@@ -618,11 +618,13 @@ int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) {
 #endif
 }
 
-void* atomic_sub_fetch_ptr(void *ptr, void* val) {
+void* atomic_sub_fetch_ptr(void *ptr, int64_t val) {
 #ifdef WINDOWS
   return interlocked_sub_fetch_ptr(ptr, val);
 #elif defined(_TD_NINGSI_60)
   return __sync_sub_and_fetch((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return __atomic_sub_fetch((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_sub_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -673,6 +675,8 @@ void* atomic_fetch_sub_ptr(void *ptr, void* val) {
   return interlocked_fetch_sub_ptr(ptr, val);
 #elif defined(_TD_NINGSI_60)
   return __sync_fetch_and_sub((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return __atomic_fetch_sub((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_fetch_sub((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -723,6 +727,8 @@ void* atomic_and_fetch_ptr(void *ptr, void *val) {
   return interlocked_and_fetch_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_and_and_fetch((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_and_fetch((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_and_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -773,6 +779,8 @@ void* atomic_fetch_and_ptr(void *ptr, void *val) {
   return interlocked_fetch_and_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_fetch_and_and((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_fetch_and((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_fetch_and((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -823,6 +831,8 @@ void* atomic_or_fetch_ptr(void *ptr, void *val) {
   return interlocked_or_fetch_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_or_and_fetch((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_or_fetch((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_or_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -873,6 +883,8 @@ void* atomic_fetch_or_ptr(void *ptr, void *val) {
   return interlocked_fetch_or_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_fetch_and_or((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_fetch_or((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_fetch_or((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -923,6 +935,8 @@ void* atomic_xor_fetch_ptr(void *ptr, void *val) {
   return interlocked_xor_fetch_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_xor_and_fetch((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_xor_fetch((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_xor_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -973,6 +987,8 @@ void* atomic_fetch_xor_ptr(void *ptr, void *val) {
   return interlocked_fetch_xor_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_fetch_and_xor((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_fetch_xor((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_fetch_xor((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif

source/os/src/osFile.c

@@ -35,6 +35,8 @@
 #include <unistd.h>
 #define LINUX_FILE_NO_TEXT_OPTION 0
 #define O_TEXT                    LINUX_FILE_NO_TEXT_OPTION
+
+#define _SEND_FILE_STEP_ 1000
 #endif
 
 #if defined(WINDOWS)
@@ -612,28 +614,34 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
 
 #elif defined(_TD_DARWIN_64)
 
-  int r = 0;
-  if (offset) {
-    r = fseek(in_file, *offset, SEEK_SET);
-    if (r == -1) return -1;
-  }
-  off_t len = size;
-  while (len > 0) {
-    char  buf[1024 * 16];
-    off_t n = sizeof(buf);
-    if (len < n) n = len;
-    size_t m = fread(buf, 1, n, in_file);
-    if (m < n) {
-      int e = ferror(in_file);
-      if (e) return -1;
-    }
-    if (m == 0) break;
-    if (m != fwrite(buf, 1, m, out_file)) {
-      return -1;
+  lseek(pFileIn->fd, (int32_t)(*offset), 0);
+  int64_t writeLen = 0;
+  uint8_t buffer[_SEND_FILE_STEP_] = {0};
+
+  for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
+    size_t rlen = read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_);
+    if (rlen <= 0) {
+      return writeLen;
+    } else if (rlen < _SEND_FILE_STEP_) {
+      write(pFileOut->fd, (void *)buffer, (uint32_t)rlen);
+      return (int64_t)(writeLen + rlen);
+    } else {
+      write(pFileOut->fd, (void *)buffer, _SEND_FILE_STEP_);
+      writeLen += _SEND_FILE_STEP_;
     }
-    len -= m;
   }
-  return size - len;
+
+  int64_t remain = size - writeLen;
+  if (remain > 0) {
+    size_t rlen = read(pFileIn->fd, (void *)buffer, (size_t)remain);
+    if (rlen <= 0) {
+      return writeLen;
+    } else {
+      write(pFileOut->fd, (void *)buffer, (uint32_t)remain);
+      writeLen += remain;
+    }
+  }
+  return writeLen;
 
 #else
 

source/os/src/osMemory.c

@@ -14,7 +14,11 @@
  */
 
 #define ALLOW_FORBID_FUNC
+#ifdef _TD_DARWIN_64
+#include <malloc/malloc.h>
+#else
 #include <malloc.h>
+#endif
 #include "os.h"
 
 #if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE)
@@ -323,6 +327,8 @@ int32_t taosMemorySize(void *ptr) {
 #else
 #ifdef WINDOWS
   return _msize(ptr);
+#elif defined(_TD_DARWIN_64)
+  return malloc_size(ptr);
 #else
   return malloc_usable_size(ptr);
 #endif

source/os/src/osSignal.c

@@ -73,6 +73,10 @@ void taosIgnSignal(int32_t signum) { signal(signum, SIG_IGN); }
 
 void taosDflSignal(int32_t signum) { signal(signum, SIG_DFL); }
 
-void taosKillChildOnParentStopped() { prctl(PR_SET_PDEATHSIG, SIGKILL); }
+void taosKillChildOnParentStopped() { 
+#ifndef _TD_DARWIN_64
+  prctl(PR_SET_PDEATHSIG, SIGKILL); 
+#endif
+}
 
 #endif

source/os/src/osSocket.c

@@ -49,6 +49,14 @@
 #define INVALID_SOCKET -1
 #endif
 
+typedef struct TdSocket {
+#if SOCKET_WITH_LOCK
+  TdThreadRwlock rwlock;
+#endif
+  int      refId;
+  SocketFd fd;
+} * TdSocketPtr, TdSocket;
+
 typedef struct TdSocketServer {
 #if SOCKET_WITH_LOCK
   TdThreadRwlock rwlock;
@@ -1029,60 +1037,6 @@ int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *a
   return getsockname(pSocket->fd, destAddr, addrLen);
 }
 
-TdEpollPtr taosCreateEpoll(int32_t size) {
-  EpollFd fd = -1;
-#ifdef WINDOWS
-  assert(0);
-#else
-  fd = epoll_create(size);
-#endif
-  if (fd < 0) {
-    return NULL;
-  }
-
-  TdEpollPtr pEpoll = (TdEpollPtr)taosMemoryMalloc(sizeof(TdEpoll));
-  if (pEpoll == NULL) {
-    taosCloseSocketNoCheck1(fd);
-    return NULL;
-  }
-  pEpoll->fd = fd;
-  pEpoll->refId = 0;
-  return pEpoll;
-}
-int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event) {
-  int32_t code = -1;
-  if (pEpoll == NULL || pEpoll->fd < 0) {
-    return -1;
-  }
-#ifdef WINDOWS
-  assert(0);
-#else
-  code = epoll_ctl(pEpoll->fd, epollOperate, pSocket->fd, event);
-#endif
-  return code;
-}
-int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout) {
-  int32_t code = -1;
-  if (pEpoll == NULL || pEpoll->fd < 0) {
-    return -1;
-  }
-#ifdef WINDOWS
-  assert(0);
-#else
-  code = epoll_wait(pEpoll->fd, event, maxEvents, timeout);
-#endif
-  return code;
-}
-int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) {
-  int32_t code;
-  if (ppEpoll == NULL || *ppEpoll == NULL || (*ppEpoll)->fd < 0) {
-    return -1;
-  }
-  code = taosCloseSocketNoCheck1((*ppEpoll)->fd);
-  (*ppEpoll)->fd = -1;
-  taosMemoryFree(*ppEpoll);
-  return code;
-}
 /*
  * Set TCP connection timeout per-socket level.
  * ref [https://github.com/libuv/help/issues/54]
@@ -1100,6 +1054,11 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) {
   if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) {
     return -1;
   }
+#elif defined(_TD_DARWIN_64)
+  uint32_t conn_timeout_ms = timeout * 1000;
+  if (0 != setsockopt(fd, IPPROTO_TCP, TCP_CONNECTIONTIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) {
+    return -1;
+  }
 #else  // Linux like systems
   uint32_t conn_timeout_ms = timeout * 1000;
   if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) {

source/os/src/osString.c

@@ -196,7 +196,7 @@ int32_t taosUcs4len(TdUcs4 *ucs4) {
 }
 
 //dst buffer size should be at least 2*len + 1
-int32_t taosHexEncode(const char *src, char *dst, int32_t len) {
+int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len) {
   if (!dst) {
     return -1;
   }

source/os/src/osThread.c

@@ -157,9 +157,9 @@ int32_t taosThreadKill(TdThread thread, int32_t sig) {
   return pthread_kill(thread, sig);
 }
 
-int32_t taosThreadMutexConsistent(TdThreadMutex* mutex) {
-  return pthread_mutex_consistent(mutex);
-}
+// int32_t taosThreadMutexConsistent(TdThreadMutex* mutex) {
+//   return pthread_mutex_consistent(mutex);
+// }
 
 int32_t taosThreadMutexDestroy(TdThreadMutex * mutex) {
   return pthread_mutex_destroy(mutex);
@@ -173,9 +173,9 @@ int32_t taosThreadMutexLock(TdThreadMutex * mutex) {
   return pthread_mutex_lock(mutex);
 }
 
-int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime) {
-  return pthread_mutex_timedlock(mutex, abstime);
-}
+// int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime) {
+//   return pthread_mutex_timedlock(mutex, abstime);
+// }
 
 int32_t taosThreadMutexTryLock(TdThreadMutex * mutex) {
   return pthread_mutex_trylock(mutex);
@@ -193,9 +193,9 @@ int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr * attr, int32_t *p
   return pthread_mutexattr_getpshared(attr, pshared);
 }
 
-int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust) {
-  return pthread_mutexattr_getrobust(attr, robust);
-}
+// int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust) {
+//   return pthread_mutexattr_getrobust(attr, robust);
+// }
 
 int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr * attr, int32_t *kind) {
   return pthread_mutexattr_gettype(attr, kind);
@@ -209,9 +209,9 @@ int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr * attr, int32_t pshared)
   return pthread_mutexattr_setpshared(attr, pshared);
 }
 
-int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust) {
-  return pthread_mutexattr_setrobust(attr, robust);
-}
+// int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust) {
+//   return pthread_mutexattr_setrobust(attr, robust);
+// }
 
 int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr * attr, int32_t kind) {
   return pthread_mutexattr_settype(attr, kind);
@@ -233,13 +233,13 @@ int32_t taosThreadRwlockRdlock(TdThreadRwlock * rwlock) {
   return pthread_rwlock_rdlock(rwlock);
 }
 
-int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime) {
-  return pthread_rwlock_timedrdlock(rwlock, abstime);
-}
+// int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime) {
+//   return pthread_rwlock_timedrdlock(rwlock, abstime);
+// }
 
-int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime) {
-  return pthread_rwlock_timedwrlock(rwlock, abstime);
-}
+// int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime) {
+//   return pthread_rwlock_timedwrlock(rwlock, abstime);
+// }
 
 int32_t taosThreadRwlockTryRdlock(TdThreadRwlock * rwlock) {
   return pthread_rwlock_tryrdlock(rwlock);
@@ -303,7 +303,7 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock) {
 
 int32_t taosThreadSpinInit(TdThreadSpinlock * lock, int32_t pshared) {
 #ifdef TD_USE_SPINLOCK_AS_MUTEX
-  assert(pshared == NULL);
+  assert(pshared == 0);
   return pthread_mutex_init((pthread_mutex_t*)lock, NULL);
 #else
   return pthread_spin_init((pthread_spinlock_t*)lock, pshared);

source/util/src/tcache.c

@@ -417,8 +417,8 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
 
   if (pNode == NULL) {
     pushfrontNodeInEntryList(pe, pNode1);
-    atomic_add_fetch_64(&pCacheObj->numOfElems, 1);
-    atomic_add_fetch_64(&pCacheObj->sizeInBytes, pNode1->size);
+    atomic_add_fetch_ptr(&pCacheObj->numOfElems, 1);
+    atomic_add_fetch_ptr(&pCacheObj->sizeInBytes, pNode1->size);
     uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64
            ", totalNum:%d sizeInBytes:%" PRId64 "bytes size:%" PRId64 "bytes",
            pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)pCacheObj->numOfElems,
@@ -667,7 +667,7 @@ void doTraverseElems(SCacheObj *pCacheObj, bool (*fp)(void *param, SCacheNode *p
         pEntry->next = next;
         pEntry->num -= 1;
 
-        atomic_sub_fetch_64(&pCacheObj->numOfElems, 1);
+        atomic_sub_fetch_ptr(&pCacheObj->numOfElems, 1);
         pNode = next;
       }
     }

source/util/src/tcompare.c

@@ -56,11 +56,11 @@ int32_t setChkNotInBytes8(const void *pLeft, const void *pRight) {
 }
 
 int32_t compareChkInString(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, varDataTLen(pLeft)) ? 1 : 0;
 }
 
 int32_t compareChkNotInString(const void *pLeft, const void *pRight) {
-  return NULL == taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
+  return NULL == taosHashGet((SHashObj *)pRight, pLeft, varDataTLen(pLeft)) ? 1 : 0;
 }
 
 int32_t compareInt8Val(const void *pLeft, const void *pRight) {

source/util/src/terror.c

@@ -590,6 +590,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE,         "No tsma index in ca
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV,               "Invalid rsma env")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT,              "Invalid rsma state")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_QTASKINFO_CREATE,          "Rsma qtaskinfo creation error")
+TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FILE_CORRUPTED,            "Rsma file corrupted")
+
 
 //tq
 TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET,         "No committed offset")

source/util/src/ttimer.c

@@ -132,7 +132,7 @@ static timer_map_t timerMap;
 static uintptr_t getNextTimerId() {
   uintptr_t id;
   do {
-    id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, (void*)1);
+    id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, 1);
   } while (id == 0);
   return id;
 }

tests/pytest/util/common.py

@@ -663,6 +663,12 @@ class TDCom:
         else:
             tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}")
             
+    def killProcessor(self, processorName):        
+        if (platform.system().lower() == 'windows'):
+            os.system("TASKKILL /F /IM %s.exe"%processorName)
+        else:
+            os.system('pkill %s'%processorName)        
+
 
 def is_json(msg):
     if isinstance(msg, str):

tests/script/jenkins/basic.txt

@@ -93,6 +93,7 @@
 ./test.sh -f tsim/stream/basic0.sim
 ./test.sh -f tsim/stream/basic1.sim
 ./test.sh -f tsim/stream/basic2.sim
+./test.sh -f tsim/stream/drop_stream.sim
 ./test.sh -f tsim/stream/distributeInterval0.sim
 # ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim
 # ./test.sh -f tsim/stream/distributesession0.sim
@@ -105,6 +106,7 @@
 ./test.sh -f tsim/stream/partitionby1.sim
 ./test.sh -f tsim/stream/schedSnode.sim
 ./test.sh -f tsim/stream/windowClose.sim
+./test.sh -f tsim/stream/ignoreExpiredData.sim
 
 # ---- transaction
  ./test.sh -f tsim/trans/lossdata1.sim
@@ -159,6 +161,7 @@
 #./test.sh -f tsim/mnode/basic1.sim -m
 
 # --- sma
+./test.sh -f tsim/sma/drop_sma.sim
 ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
 ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
 

tests/script/sh/deploy.sh

@@ -135,7 +135,7 @@ echo "jniDebugFlag           143"                >> $TAOS_CFG
 echo "qDebugFlag             143"                >> $TAOS_CFG
 echo "rpcDebugFlag           143"                >> $TAOS_CFG
 echo "tmrDebugFlag           131"                >> $TAOS_CFG
-echo "uDebugFlag             143"                >> $TAOS_CFG
+echo "uDebugFlag             131"                >> $TAOS_CFG
 echo "sDebugFlag             143"                >> $TAOS_CFG
 echo "wDebugFlag             143"                >> $TAOS_CFG
 echo "numOfLogLines          20000000"           >> $TAOS_CFG

tests/script/tsim/sma/drop_sma.sim

@@ -64,12 +64,59 @@ sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) in
 print --> drop stb
 sql drop table stb; 
 
+print ========== step5 
+sql drop database if exists db;
+sql create database db duration 300;
+sql use db;
+sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint, c_float float, c_double double, c_bool bool, c_binary binary(16), c_nchar nchar(32), c_ts timestamp, c_tint_un tinyint unsigned, c_sint_un smallint unsigned, c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);
+sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
+
+print ========== step6 repeat
+sql drop database if exists db;
+sql create database db duration 300;
+sql use db;
+sql create table stb1(ts timestamp, c_int int, c_bint bigint ) tags (t_int int);
+sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
+
+print ========== step7
+sql drop database if exists db;
+sql create database db duration 300;
+sql use db;
+sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);
+
+sql create table ct1 using stb1 tags ( 1 );
+sql create table ct2 using stb1 tags ( 2 );
+sql create table ct3 using stb1 tags ( 3 );
+sql create table ct4 using stb1 tags ( 4 );
+
+sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
+sql CREATE SMA INDEX sma_index_2 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) max_delay 6m;
+sql CREATE SMA INDEX sma_index_3 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) watermark 5s max_delay 6m;
+
+sql DROP INDEX sma_index_1 ;
+sql DROP INDEX sma_index_2 ;
+sql DROP INDEX sma_index_3 ;
+
+print ========== step8
+sql drop database if exists db;
+sql create database db duration 300;
+sql use db;
+sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);
+
+sql create table ct1 using stb1 tags ( 1 );
+sql create table ct2 using stb1 tags ( 2 );
+sql create table ct3 using stb1 tags ( 3 );
+sql create table ct4 using stb1 tags ( 4 );
+
+sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
+sql CREATE SMA INDEX sma_index_2 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) max_delay 6m;
+sql CREATE SMA INDEX sma_index_3 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) watermark 5s max_delay 6m;
+
+sql DROP INDEX sma_index_1 ;
+sql DROP INDEX sma_index_2 ;
+sql DROP INDEX sma_index_3 ;
 
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
 system sh/exec.sh -n dnode4 -s stop  -x SIGINT
\ No newline at end of file
-system sh/exec.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/tsim/stream/distributeInterval0.sim

@@ -83,9 +83,9 @@ sql insert into ts3 values(1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0)
 
 $loop_count = 0
 loop1:
+sleep 300
 sql select * from streamtST1;
 
-sleep 300
 $loop_count = $loop_count + 1
 if $loop_count == 10 then
   return -1

tests/script/tsim/stream/distributeSession0.sim

@@ -10,6 +10,30 @@ sql create dnode $hostname2 port 7200
 
 system sh/exec.sh -n dnode2 -s start
 
+print ===== step1
+$x = 0
+step1:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 2 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+
+print ===== step2
+
 sql create database test  vgroups 4;
 sql use test;
 sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);

tests/script/tsim/stream/ignoreExpiredData.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+
+system sh/exec.sh -n dnode1 -s start 
+sleep 50
+sql connect
+
+sql create dnode $hostname2 port 7200
+
+system sh/exec.sh -n dnode2 -s start
+
+print ===== step1
+$x = 0
+step1:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 2 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+
+print ===== step2
+
+print =============== create database
+sql create database test vgroups 1
+sql show databases
+if $rows != 3 then
+  return -1
+endi
+
+print $data00 $data01 $data02
+
+sql use test
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams1 trigger at_once IGNORE EXPIRED into streamt1 as select  _wstartts, count(*) c1, sum(a) c3 from t1 interval(10s);
+sql create stream streams2 trigger at_once IGNORE EXPIRED into streamt2 as select  _wstartts, count(*) c1, sum(a) c3 from t1 session(ts,10s);
+sql create stream streams3 trigger at_once IGNORE EXPIRED into streamt3 as select  _wstartts, count(*) c1, sum(a) c3 from t1 state_window(a);
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.1);
+sql insert into t1 values(1648791233002,2,2,3,2.1);
+sql insert into t1 values(1648791243003,2,2,3,3.1);
+sql insert into t1 values(1648791200000,4,2,3,4.1);
+
+$loop_count = 0
+loop1:
+sleep 300
+sql select * from streamt1;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 4 then
+  print =====rows=$rows
+  goto loop1
+endi
+
+$loop_count = 0
+loop2:
+sleep 300
+sql select * from streamt2;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 4 then
+  print =====rows=$rows
+  goto loop2
+endi
+
+$loop_count = 0
+loop3:
+sleep 300
+sql select * from streamt3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 2 then
+  print =====rows=$rows
+  goto loop3
+endi
+
+
+print =============== create database
+sql create database test1 vgroups 4
+sql show databases
+
+print ======database=$rows
+
+sql use test1
+
+sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create table ts2 using st tags(2,2,2);
+sql create stream stream_t1 trigger at_once IGNORE EXPIRED into streamtST1 as select  _wstartts, count(*) c1, count(a) c2 , sum(a) c3 ,  max(b)  c5, min(c) c6 from st interval(10s) ;
+sql create stream stream_t2 trigger at_once IGNORE EXPIRED into streamtST2 as select  _wstartts, count(*) c1, count(a) c2 , sum(a) c3 ,  max(b)  c5, min(c) c6 from st session(ts, 10s) ;
+sql insert into ts1 values(1648791211000,1,2,3);
+sql insert into ts1 values(1648791222001,2,2,3);
+sql insert into ts2 values(1648791211000,1,2,3);
+sql insert into ts2 values(1648791222001,2,2,3);
+
+$loop_count = 0
+loop4:
+sleep 300
+sql select * from streamtST1;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop4
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop4
+endi
+
+$loop_count = 0
+loop5:
+sleep 300
+sql select * from streamtST2;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop5
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop5
+endi
+
+system sh/stop_dnodes.sh
\ No newline at end of file

tests/script/tsim/tmq/snapshot1.sim

@@ -25,7 +25,7 @@ $rowsPerCtb = 10
 $tstart     = 1640966400000  # 2022-01-01 00:00:00.000
 #---- global parameters end ----#
 
-$pullDelay    = 5
+$pullDelay    = 2
 $ifcheckdata  = 1
 $ifmanualcommit = 1
 $showMsg      = 1

tests/system-test/2-query/Timediff.py

@@ -72,12 +72,12 @@ class TDTestCase:
         tdSql.query("select timediff(1,0,1a) from db.ntb")
         tdSql.checkRows(3)
         tdSql.checkData(0,0,1000)
-        tdSql.query("select timediff(1,0,1u) from ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,1000000)
-        tdSql.query("select timediff(1,0,1u) from db.ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,1000000)
+        tdSql.error("select timediff(1,0,1u) from ntb")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,1000000)
+        tdSql.error("select timediff(1,0,1u) from db.ntb")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,1000000)
 
         tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00') from stb")
         tdSql.checkRows(3)
@@ -116,12 +116,12 @@ class TDTestCase:
         tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1a) from db.stb")
         tdSql.checkRows(3)
         tdSql.checkData(0,0,86400000)
-        tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,86400000000)
-        tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from db.stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,86400000000)
+        tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from stb")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,86400000000)
+        tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from db.stb")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,86400000000)
 
 
         tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00') from stb_1")
@@ -164,12 +164,12 @@ class TDTestCase:
         tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1a) from db.stb_1")
         tdSql.checkRows(3)
         tdSql.checkData(0,0,43200000)
-        tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from stb_1")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,43200000000)
-        tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from db.stb_1")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,43200000000)
+        tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from stb_1")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,43200000000)
+        tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from db.stb_1")
+        #tdSql.checkRows(3)
+        #tdSql.checkData(0,0,43200000000)
 
         tdSql.query("select timediff('a','b') from stb")
         tdSql.checkRows(3)

tests/system-test/2-query/elapsed.py

@@ -1488,8 +1488,7 @@ class TDTestCase:
 
         tdSql.query('select elapsed(ts,10s) from  ( select * from sub_table1_1   where ts>="2015-01-01 00:00:00.000"  and ts < "2015-01-01 00:10:00.000") session(ts,1w) ; ')
 
-        tdSql.query('select elapsed(ts,10s) from  ( select ts ,q_int from sub_table1_1   where ts>="2015-01-01 00:00:00.000"  and ts < "2015-01-01 00:10:00.000") session(ts,1w) ; ')
-    
+        tdSql.error('select elapsed(ts,10s) from  ( select ts ,q_int from sub_table1_1   where ts>="2015-01-01 00:00:00.000"  and ts < "2015-01-01 00:10:00.000") session(ts,1w) ; ')
         tdSql.error('select elapsed(ts,10s) from sub_table1_1   where ts>="2015-01-01 00:00:00.000"  and ts < "2015-01-01 00:10:00.000" interval(20s) fill (next) session(ts,1w) ; ')
 
         tdSql.query('select elapsed(ts,10s) from sub_empty_1  where ts>="2015-01-01 00:00:00.000"  and ts < "2015-01-01 00:10:00.000"  session(ts,1w) ; ')

tests/system-test/2-query/function_stateduration.py

@@ -348,10 +348,9 @@ class TDTestCase:
         tdSql.execute(" use db ")
         tdSql.error("select stateduration(c1,'GT',1,1b) from ct1")
         tdSql.error("select stateduration(c1,'GT',1,1u) from ct1")
+        tdSql.error("select stateduration(c1,'GT',1,1000s) from t1")
         tdSql.query("select stateduration(c1,'GT',1,1s) from t1")
         tdSql.checkData(10,0,63072035)
-        tdSql.query("select stateduration(c1,'GT',1,1000s) from t1")
-        tdSql.checkData(10,0,int(63072035/1000))
         tdSql.query("select stateduration(c1,'GT',1,1m) from t1")
         tdSql.checkData(10,0,int(63072035/60))
         tdSql.query("select stateduration(c1,'GT',1,1h) from t1")

tests/system-test/2-query/json_tag.py

@@ -675,7 +675,7 @@ class TDTestCase:
         tdSql.checkRows(3)
         tdSql.query("select TO_UNIXTIMESTAMP(datastr) from jsons1 where jtag->'tag1'>1;")
         tdSql.checkRows(3)
-        tdSql.query("select TIMETRUNCATE(ts,1u) from jsons1 where jtag->'tag1'>1;")
+        tdSql.query("select TIMETRUNCATE(ts,1s) from jsons1 where jtag->'tag1'>1;")
         tdSql.checkRows(3)
         tdSql.query("select TIMEDIFF(ts,_c0) from jsons1 where jtag->'tag1'>1;")
         tdSql.checkRows(3)

tests/system-test/2-query/statecount.py

@@ -343,10 +343,9 @@ class TDTestCase:
         tdSql.execute(" use db ")
         tdSql.error("select stateduration(c1,'GT',1,1b) from ct1")
         tdSql.error("select stateduration(c1,'GT',1,1u) from ct1")
+        tdSql.error("select stateduration(c1,'GT',1,1000s) from t1")
         tdSql.query("select stateduration(c1,'GT',1,1s) from t1")
         tdSql.checkData(10,0,63072035)
-        tdSql.query("select stateduration(c1,'GT',1,1000s) from t1")
-        tdSql.checkData(10,0,int(63072035/1000))
         tdSql.query("select stateduration(c1,'GT',1,1m) from t1")
         tdSql.checkData(10,0,int(63072035/60))
         tdSql.query("select stateduration(c1,'GT',1,1h) from t1")

tests/system-test/2-query/timetruncate.py

@@ -37,8 +37,8 @@ class TDTestCase:
 
         tdSql.query("select timetruncate(1,1d) from ntb")
         tdSql.checkRows(10)
-        tdSql.query("select timetruncate(1,1u) from ntb")
-        tdSql.checkRows(10)
+        tdSql.error("select timetruncate(1,1u) from ntb")
+        #tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1a) from ntb")
         tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1m) from ntb")
@@ -97,8 +97,8 @@ class TDTestCase:
 
         tdSql.query("select timetruncate(1,1d) from stb")
         tdSql.checkRows(10)
-        tdSql.query("select timetruncate(1,1u) from stb")
-        tdSql.checkRows(10)
+        tdSql.error("select timetruncate(1,1u) from stb")
+        #tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1a) from stb")
         tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1m) from stb")
@@ -156,8 +156,8 @@ class TDTestCase:
 
         tdSql.query("select timetruncate(1,1d) from stb_1")
         tdSql.checkRows(10)
-        tdSql.query("select timetruncate(1,1u) from stb_1")
-        tdSql.checkRows(10)
+        tdSql.error("select timetruncate(1,1u) from stb_1")
+        #tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1a) from stb_1")
         tdSql.checkRows(10)
         tdSql.query("select timetruncate(1,1m) from stb_1")

tests/system-test/7-tmq/stbFilter.py

@@ -20,18 +20,54 @@ class TDTestCase:
         tdSql.init(conn.cursor())
         #tdSql.init(conn.cursor(), logSql)  # output sql.txt file
 
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   100,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1}
+
+        tmqCom.initConsumerTable()
+        tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica'])
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        return
+
     def tmqCase1(self):
         tdLog.printNoPrefix("======== test case 1: ")
-        paraDict = {'dbName':     'db1',
+        paraDict = {'dbName':     'dbt',
                     'dropFlag':   1,
                     'event':      '',
                     'vgroups':    4,
+                    'replica':    1,
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}],
                     'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
                     'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
                     'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   100,
@@ -43,13 +79,6 @@ class TDTestCase:
         topicNameList = ['topic1', 'topic2', 'topic3']
         expectRowsList = []
         tmqCom.initConsumerTable()
-        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
-        tdLog.info("create stb")
-        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        tdLog.info("create ctb")
-        tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
-        tdLog.info("insert data")
-        tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
         
         tdLog.info("create topics from stb with filter")
         queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 4 == 0" %(paraDict['dbName'], paraDict['stbName'])
@@ -122,9 +151,9 @@ class TDTestCase:
         tdLog.info("wait the consume result") 
         expectRows = 1
         resultList = tmqCom.selectConsumeResult(expectRows)
-        # if expectRowsList[2] != resultList[0]:
-        #     tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
-        #     tdLog.exit("2 tmq consume rows error!")
+        if expectRowsList[2] != resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
+            tdLog.exit("2 tmq consume rows error!")
 
         time.sleep(10)        
         for i in range(len(topicNameList)):
@@ -134,16 +163,18 @@ class TDTestCase:
 
     def tmqCase2(self):
         tdLog.printNoPrefix("======== test case 2: ")
-        paraDict = {'dbName':     'db2',
+        paraDict = {'dbName':     'dbt',
                     'dropFlag':   1,
                     'event':      '',
                     'vgroups':    4,
+                    'replica':    1,
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
                     'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}],
                     'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
                     'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
                     'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   100,
@@ -155,13 +186,6 @@ class TDTestCase:
         topicNameList = ['topic1', 'topic2', 'topic3']
         expectRowsList = []
         tmqCom.initConsumerTable()
-        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
-        tdLog.info("create stb")
-        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        tdLog.info("create ctb")
-        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'])
-        tdLog.info("insert data")
-        tmqCom.insert_data_1(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
 
         tdLog.info("create topics from stb with filter")
         # sqlString = "create topic %s as select ts, sin(c1), pow(c2,3) from %s.%s where c2 >= 0" %(topicNameList[0], paraDict['dbName'], paraDict['stbName'])
@@ -247,6 +271,7 @@ class TDTestCase:
 
     def run(self):
         tdSql.prepare()
+        self.prepareTestEnv()
         self.tmqCase1()
         self.tmqCase2()
 

tests/system-test/7-tmq/tmqCommon.py

@@ -170,33 +170,42 @@ class TMQCom:
         tdLog.debug("complete to create database %s"%(dbName))
         return
 
+    # self.create_stable() and self.create_ctable() and self.insert_data_interlaceByMultiTbl() : The three functions are matched
+    # schema: (ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32))
     def create_stable(self,tsql, dbName,stbName):
-        tsql.execute("create table if not exists %s.%s (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbName, stbName))
+        schemaString = "(ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32))"
+        tsql.execute("create table if not exists %s.%s %s"%(dbName, stbName, schemaString))
         tdLog.debug("complete to create %s.%s" %(dbName, stbName))
         return
 
     def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0):
-        tsql.execute("use %s" %dbName)
+        # tsql.execute("use %s" %dbName)
         pre_create = "create table"
         sql = pre_create
         #tdLog.debug("doing create one  stable %s and %d  child table in %s  ..." %(stbname, count ,dbname))
+        batchNum = 10
+        tblBatched  = 0
         for i in range(ctbNum):
-            tagValue = 'beijing'
+            tagBinaryValue = 'beijing'
             if (i % 2 == 0):
-                tagValue = 'shanghai'
+                tagBinaryValue = 'shanghai'
             elif (i % 3 == 0):
-                tagValue = 'changsha'
+                tagBinaryValue = 'changsha'
             
-            sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i+ctbStartIdx,stbName,i+ctbStartIdx+1, tagValue)
-            if (i > 0) and (i%100 == 0):
+            sql += " %s.%s%d using %s.%s tags(%d, %d, %d, '%s', '%s')"%(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,tagBinaryValue,tagBinaryValue)
+            tblBatched += 1
+            if (i == ctbNum-1 ) or (tblBatched == batchNum):
                 tsql.execute(sql)
+                tblBatched = 0
                 sql = pre_create
+
         if sql != pre_create:
             tsql.execute(sql)
         
-        tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName))
+        tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName))
         return    
 
+    # schema: (ts timestamp, c1 int, c2 binary(16))
     def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
@@ -208,11 +217,14 @@ class TMQCom:
             startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
+            rowsBatched = 0
             sql += " %s%d values "%(stbName,i)
             for j in range(rowsPerTbl):
                 sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
-                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+                rowsBatched += 1
+                if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
                     tsql.execute(sql)
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
                         sql = "insert into %s%d values " %(stbName,i)
                     else:
@@ -224,6 +236,7 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return        
 
+    # schema: (ts timestamp, c1 int, c2 int, c3 binary(16))
     def insert_data_1(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
@@ -234,14 +247,17 @@ class TMQCom:
         startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
+            rowsBatched = 0
             sql += " %s%d values "%(ctbPrefix,i)
             for j in range(rowsPerTbl):
                 if (j % 2 == 0):
                     sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, j, j)
                 else:
                     sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, -j, j)
-                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+                rowsBatched += 1
+                if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
                     tsql.execute(sql)
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
                         sql = "insert into %s%d values " %(ctbPrefix,i)
                     else:
@@ -253,6 +269,7 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return
 
+    # schema: (ts timestamp, c1 int, c2 int, c3 binary(16), c4 timestamp)
     def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,ctbStartIdx=0):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
@@ -263,14 +280,17 @@ class TMQCom:
         startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
+            rowsBatched = 0
             sql += " %s%d values "%(ctbPrefix,i+ctbStartIdx)
             for j in range(rowsPerTbl):
                 if (j % 2 == 0):
                     sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, j, j)
                 else:
                     sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, -j, j)
-                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+                rowsBatched += 1
+                if (rowsBatched == batchNum) or (j == rowsPerTbl - 1):
                     tsql.execute(sql)
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
                         sql = "insert into %s%d values " %(ctbPrefix,i+ctbStartIdx)
                     else:
@@ -282,7 +302,8 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return
 
-    def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0):
+    # schema: (ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32))
+    def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0,ctbStartIdx=0):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
         pre_insert = "insert into "
@@ -300,12 +321,19 @@ class TMQCom:
         rowsOfCtb = 0
         while rowsOfCtb < rowsPerTbl:
             for i in range(ctbNum):
-                sql += " %s.%s_%d values "%(dbName,ctbPrefix,i)
+                sql += " %s.%s%d values "%(dbName,ctbPrefix,i+ctbStartIdx)
+                rowsBatched = 0
                 for k in range(batchNum):
-                    sql += "(%d, %d, 'tmqrow_%d') "%(startTs + ctbDict[i], ctbDict[i], ctbDict[i])
+                    if (k % 2 == 0):
+                        sql += "(%d, %d, %d, %d, 'binary_%d', 'nchar_%d', now) "%(startTs+ctbDict[i], ctbDict[i],ctbDict[i], ctbDict[i],i+ctbStartIdx,k)
+                    else:
+                        sql += "(%d, %d, %d, %d, 'binary_%d', 'nchar_%d', now) "%(startTs+ctbDict[i],-ctbDict[i],ctbDict[i],-ctbDict[i],i+ctbStartIdx,k)
+
+                    rowsBatched += 1
                     ctbDict[i] += 1
-                    if (0 == ctbDict[i]%batchNum) or (ctbDict[i] == rowsPerTbl):
+                    if (rowsBatched == batchNum) or (ctbDict[i] == rowsPerTbl):
                         tsql.execute(sql)
+                        rowsBatched = 0
                         sql = "insert into "
                         break
             rowsOfCtb = ctbDict[0]
@@ -313,7 +341,18 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return
 
-    def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0):
+    def threadFunctionForInsertByInterlace(self, **paraDict):
+        # create new connector for new tdSql instance in my thread
+        newTdSql = tdCom.newTdSql()
+        self.insert_data_interlaceByMultiTbl(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"],paraDict["ctbStartIdx"])
+        return
+
+    def asyncInsertDataByInterlace(self, paraDict):
+        pThread = threading.Thread(target=self.threadFunctionForInsertByInterlace, kwargs=paraDict)
+        pThread.start()
+        return pThread
+
+    def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0,ctbStartIdx=0):
         tdLog.debug("start to insert data wiht auto create child table ............")
         tsql.execute("use %s" %dbName)
         pre_insert = "insert into "
@@ -324,17 +363,17 @@ class TMQCom:
             startTs = int(round(t * 1000))
 
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
-        rowsOfSql = 0        
+        rowsBatched = 0        
         for i in range(ctbNum):
-            sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i,dbName,stbName,i)
+            sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i)
             for j in range(rowsPerTbl):
                 sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
-                rowsOfSql += 1
-                if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)):
+                rowsBatched += 1
+                if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
                     tsql.execute(sql)
-                    rowsOfSql = 0
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
-                        sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i,dbName,stbName,i)
+                        sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i)
                     else:
                         sql = "insert into "
         #end sql

tests/system-test/7-tmq/tmqConsFromTsdb.py

@@ -16,7 +16,7 @@ sys.path.append("./7-tmq")
 from tmqCommon import *
 
 class TDTestCase:
-    def __int__(self):
+    def __init__(self):
         self.vgroups    = 1
         self.ctbNum     = 10
         self.rowsPerTbl = 10000
@@ -25,8 +25,8 @@ class TDTestCase:
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor(), False)
 
-    def tmqCase1(self):
-        tdLog.printNoPrefix("======== test case 1: ")
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
         paraDict = {'dbName':     'dbt',
                     'dropFlag':   1,
                     'event':      '',
@@ -34,10 +34,11 @@ class TDTestCase:
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
                     'ctbPrefix':  'ctb',
-                    'ctbNum':     1,
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   10,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
@@ -50,19 +51,52 @@ class TDTestCase:
         paraDict['ctbNum'] = self.ctbNum
         paraDict['rowsPerTbl'] = self.rowsPerTbl
         
-        topicNameList = ['topic1']
-        expectRowsList = []
         tmqCom.initConsumerTable()
         tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
         tdLog.info("create stb")
-        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
         tdLog.info("create ctb")
-        tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
         tdLog.info("insert data")
-        tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
         
+        tdLog.info("restart taosd to ensure that the data falls into the disk")
         tdDnodes.stop(1)
         tdDnodes.start(1)
+        return
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
         
         tdLog.info("create topics from stb with filter")
         queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
@@ -111,10 +145,11 @@ class TDTestCase:
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
                     'ctbPrefix':  'ctb',
-                    'ctbNum':     1,
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   10,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
@@ -130,16 +165,6 @@ class TDTestCase:
         topicNameList = ['topic1']
         expectRowsList = []
         tmqCom.initConsumerTable()
-        # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
-        # tdLog.info("create stb")
-        # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        # tdLog.info("create ctb")
-        # tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
-        # tdLog.info("insert data")
-        # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
-        
-        # tdDnodes.stop(1)
-        # tdDnodes.start(1)
         
         tdLog.info("create topics from stb with filter")
         queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
@@ -200,89 +225,9 @@ class TDTestCase:
 
         tdLog.printNoPrefix("======== test case 2 end ...... ")
 
-    def tmqCase3(self):
-        tdLog.printNoPrefix("======== test case 3: ")
-        paraDict = {'dbName':     'dbt',
-                    'dropFlag':   1,
-                    'event':      '',
-                    'vgroups':    1,
-                    'stbName':    'stb',
-                    'colPrefix':  'c',
-                    'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'ctbPrefix':  'ctb',
-                    'ctbNum':     1,
-                    'rowsPerTbl': 10000,
-                    'batchNum':   10,
-                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    'pollDelay':  -1,
-                    'showMsg':    1,
-                    'showRow':    1,
-                    'snapshot':   1}
-
-        paraDict['vgroups'] = self.vgroups
-        paraDict['ctbNum'] = self.ctbNum
-        paraDict['rowsPerTbl'] = self.rowsPerTbl
-        
-        topicNameList = ['topic1']
-        expectRowsList = []
-        tmqCom.initConsumerTable()
-        # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
-        # tdLog.info("create stb")
-        # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        # tdLog.info("create ctb")
-        # tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
-        # tdLog.info("insert data")
-        # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
-        
-        # tdDnodes.stop(1)
-        # tdDnodes.start(1)
-        
-        tdLog.info("create topics from stb with filter")
-        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
-        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
-        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
-        tdLog.info("create topic sql: %s"%sqlString)
-        tdSql.execute(sqlString)
-        tdSql.query(queryString)        
-        expectRowsList.append(tdSql.getRows())
-        totalRowsInserted = expectRowsList[0]
-
-        # init consume info, and start tmq_sim, then check consume result
-        tdLog.info("insert consume info to consume processor")
-        consumerId   = 3
-        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3)
-        topicList    = topicNameList[0]
-        ifcheckdata  = 1
-        ifManualCommit = 1
-        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
-        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-
-        consumerId   = 4
-        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3)
-        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-
-        tdLog.info("start consume processor 0")
-        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
-        tdLog.info("wait the consume result") 
-        
-        expectRows = 2
-        resultList = tmqCom.selectConsumeResult(expectRows)
-        actConsumeTotalRows = resultList[0] + resultList[1]
-        
-        if not (totalRowsInserted == actConsumeTotalRows):
-            tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
-            tdLog.exit("%d tmq consume rows error!"%consumerId)
-
-        time.sleep(10)        
-        for i in range(len(topicNameList)):
-            tdSql.query("drop topic %s"%topicNameList[i])
-
-        tdLog.printNoPrefix("======== test case 3 end ...... ")
-
     def run(self):
         tdSql.prepare()
+        self.prepareTestEnv()
         self.tmqCase1()
         self.tmqCase2()
 

tests/system-test/7-tmq/tmqConsFromTsdb1.py

+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def __init__(self):
+        self.vgroups    = 1
+        self.ctbNum     = 10
+        self.rowsPerTbl = 10000
+        
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        tdLog.info("restart taosd to ensure that the data falls into the disk")
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        return
+
+    def tmqCase3(self):
+        tdLog.printNoPrefix("======== test case 3: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+        totalRowsInserted = expectRowsList[0]
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 3
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3)
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        consumerId   = 4
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3)
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor 0")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result") 
+        
+        expectRows = 2
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        actConsumeTotalRows = resultList[0] + resultList[1]
+        
+        if not (totalRowsInserted == actConsumeTotalRows):
+            tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        time.sleep(10)        
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 3 end ...... ")
+
+    def tmqCase4(self):
+        tdLog.printNoPrefix("======== test case 4: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+        totalRowsInserted = expectRowsList[0]
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 5
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor 0")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        
+        tdLog.info("wait commit notify")
+        tmqCom.getStartCommitNotifyFromTmqsim()
+
+        tdLog.info("pkill consume processor")
+        tdCom.killProcessor("tmq_sim")
+        
+        # time.sleep(10)
+        
+        # reinit consume info, and start tmq_sim, then check consume result
+        tmqCom.initConsumerTable()
+        consumerId   = 6
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        tdLog.info("start consume processor 1")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result")
+        
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        actConsumeTotalRows = resultList[0]
+        
+        if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted):
+            tdLog.info("act consume rows: %d"%(actConsumeTotalRows))            
+            tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        time.sleep(10)        
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 4 end ...... ")
+
+    def run(self):
+        tdSql.prepare()
+        self.prepareTestEnv()
+        self.tmqCase3()
+        self.tmqCase4()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/fulltest.sh

@@ -91,7 +91,7 @@ python3 ./test.py -f 2-query/query_cols_tags_and_or.py
 # python3 ./test.py -f 2-query/nestedQuery_str.py
 
 python3 ./test.py -f 2-query/avg.py
-python3 ./test.py -f 2-query/elapsed.py
+#python3 ./test.py -f 2-query/elapsed.py
 python3 ./test.py -f 2-query/csum.py
 python3 ./test.py -f 2-query/mavg.py
 python3 ./test.py -f 2-query/diff.py
@@ -119,8 +119,8 @@ python3 ./test.py -f 2-query/queryQnode.py
 
 python3 ./test.py -f 6-cluster/5dnode1mnode.py 
 python3 ./test.py -f 6-cluster/5dnode2mnode.py  -N 5 -M 3
-python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
-python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3
+#python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
+#python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3
 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3
 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 
 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py  -N 5 -M 3
@@ -156,7 +156,8 @@ python3 ./test.py -f 7-tmq/tmqUdf.py
 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py
 python3 ./test.py -f 7-tmq/tmqShow.py
 python3 ./test.py -f 7-tmq/tmqAlterSchema.py
-
+python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py
+python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py
 
 
 #------------querPolicy  2-----------
@@ -332,3 +333,4 @@ python3 ./test.py -f 2-query/twa.py -Q  3
 python3 ./test.py -f 2-query/irate.py -Q  3
 python3 ./test.py -f 2-query/function_null.py -Q  3
 python3 ./test.py -f 2-query/queryQnode.py  -Q  3
+

tools/shell/src/shellArguments.c

@@ -13,6 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#ifdef _TD_DARWIN_64
+#include <pwd.h>
+#endif
+
 #include "shellInt.h"
 
 #define SHELL_HOST     "The auth string to use when connecting to the server."