Merge branch '3.0' into test/bug_fix_update

cmake/cmake.platform

@@ -76,15 +76,19 @@ IF ("${CPUTYPE}" STREQUAL "")
   IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)")
     MESSAGE(STATUS "The current platform is amd64")
     SET(PLATFORM_ARCH_STR "amd64")
+    SET(TD_INTEL_64 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)")
     MESSAGE(STATUS "The current platform is x86")
     SET(PLATFORM_ARCH_STR "i386")
+    SET(TD_INTEL_32 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l")
     MESSAGE(STATUS "The current platform is aarch32")
     SET(PLATFORM_ARCH_STR "arm")
+    SET(TD_ARM_32 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
     MESSAGE(STATUS "The current platform is aarch64")
     SET(PLATFORM_ARCH_STR "arm64")
+    SET(TD_ARM_64 TRUE)
   ENDIF ()
 ELSE ()
   # if generate ARM version:
@@ -92,18 +96,23 @@ ELSE ()
   IF (${CPUTYPE} MATCHES "aarch32")
     SET(PLATFORM_ARCH_STR "arm")
     MESSAGE(STATUS "input cpuType: aarch32")
+    SET(TD_ARM_32 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "aarch64")
     SET(PLATFORM_ARCH_STR "arm64")
     MESSAGE(STATUS "input cpuType: aarch64")
+    SET(TD_ARM_64 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "mips64")
     SET(PLATFORM_ARCH_STR "mips")
     MESSAGE(STATUS "input cpuType: mips64")
+    SET(TD_MIPS_64 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "x64")
     SET(PLATFORM_ARCH_STR "amd64")
     MESSAGE(STATUS "input cpuType: x64")
+    SET(TD_INTEL_64 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "x86")
     SET(PLATFORM_ARCH_STR "i386")
     MESSAGE(STATUS "input cpuType: x86")
+    SET(TD_INTEL_32 TRUE)
   ELSE ()
     MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE})
   ENDIF ()

include/common/tmsg.h

@@ -2071,9 +2071,10 @@ int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp);
 
 // TDMT_VND_ALTER_TABLE =====================
 typedef struct {
-  char*  tbName;
-  int8_t action;
-  char*  colName;
+  char*   tbName;
+  int8_t  action;
+  char*   colName;
+  int32_t colId;
   // TSDB_ALTER_TABLE_ADD_COLUMN
   int8_t  type;
   int8_t  flags;

include/common/ttime.h

@@ -63,12 +63,13 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) {
                       : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000
                                                                  : 1000000000;
   time_t     t = taosTime(NULL);
-  struct tm* tm = taosLocalTime(&t, NULL);
-  tm->tm_hour = 0;
-  tm->tm_min = 0;
-  tm->tm_sec = 0;
+  struct tm  tm;
+  taosLocalTime(&t, &tm);
+  tm.tm_hour = 0;
+  tm.tm_min = 0;
+  tm.tm_sec = 0;
 
-  return (int64_t)taosMktime(tm) * factor;
+  return (int64_t)taosMktime(&tm) * factor;
 }
 
 int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision);

include/libs/nodes/plannodes.h

@@ -26,6 +26,12 @@ extern "C" {
 
 #define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN
 
+typedef enum EDataOrderLevel {
+  DATA_ORDER_LEVEL_NONE = 1,
+  DATA_ORDER_LEVEL_IN_BLOCK,
+  DATA_ORDER_LEVEL_IN_GROUP
+} EDataOrderLevel;
+
 typedef struct SLogicNode {
   ENodeType          type;
   SNodeList*         pTargets;  // SColumnNode
@@ -36,6 +42,8 @@ typedef struct SLogicNode {
   uint8_t            precision;
   SNode*             pLimit;
   SNode*             pSlimit;
+  EDataOrderLevel    requireDataOrder;  // requirements for input data
+  EDataOrderLevel    resultDataOrder;   // properties of the output data
 } SLogicNode;
 
 typedef enum EScanType {
@@ -78,7 +86,7 @@ typedef struct SScanLogicNode {
   SNodeList*    pGroupTags;
   bool          groupSort;
   int8_t        cacheLastMode;
-  bool          hasNormalCols; // neither tag column nor primary key tag column
+  bool          hasNormalCols;  // neither tag column nor primary key tag column
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -317,6 +325,7 @@ typedef STableScanPhysiNode SStreamScanPhysiNode;
 typedef struct SProjectPhysiNode {
   SPhysiNode node;
   SNodeList* pProjections;
+  bool       mergeDataBlock;
 } SProjectPhysiNode;
 
 typedef struct SIndefRowsFuncPhysiNode {

source/client/src/tmq.c

@@ -363,7 +363,11 @@ tmq_list_t* tmq_list_new() {
 
 int32_t tmq_list_append(tmq_list_t* list, const char* src) {
   SArray* container = &list->container;
-  char*   topic = strdup(src);
+  if (src == NULL || src[0] == 0) return -1;
+  char* topic = strdup(src);
+  if (topic[0] != '`') {
+    strtolower(topic, src);
+  }
   if (taosArrayPush(container, &topic) == NULL) return -1;
   return 0;
 }

source/common/src/tdatablock.c

@@ -228,7 +228,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui
   uint32_t finalNumOfRows = numOfRow1 + numOfRow2;
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     // Handle the bitmap
-    if (finalNumOfRows > *capacity || numOfRow1 == 0) {
+    if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) {
       char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
       if (p == NULL) {
         return TSDB_CODE_OUT_OF_MEMORY;
@@ -262,7 +262,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui
     memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len);
     pColumnInfoData->varmeta.length = len + oldLen;
   } else {
-    if (finalNumOfRows > *capacity || numOfRow1 == 0) {
+    if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) {
       ASSERT(finalNumOfRows * pColumnInfoData->info.bytes);
       char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes);
       if (tmp == NULL) {
@@ -1661,9 +1661,6 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
     }
     */
 
-#ifdef WINDOWS
-  if (tt < 0) tt = 0;
-#endif
   if (tt <= 0 && ms < 0) {
     tt--;
     if (precision == TSDB_TIME_PRECISION_NANO) {
@@ -1674,9 +1671,9 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
       ms += 1000;
     }
   }
-
-  struct tm* ptm = taosLocalTime(&tt, NULL);
-  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
+  struct tm ptm = {0};
+  taosLocalTime(&tt, &ptm);
+  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
 
   if (precision == TSDB_TIME_PRECISION_NANO) {
     sprintf(buf + pos, ".%09d", ms);

source/common/src/ttime.c

@@ -902,7 +902,7 @@ const char* fmtts(int64_t ts) {
 
 void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) {
   char       ts[40] = {0};
-  struct tm* ptm;
+  struct tm  ptm;
 
   int32_t fractionLen;
   char*   format = NULL;
@@ -939,10 +939,10 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)
       assert(false);
   }
 
-  ptm = taosLocalTime(&quot, NULL);
-  int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm);
+  taosLocalTime(&quot, &ptm);
+  int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm);
   length += snprintf(ts + length, fractionLen, format, mod);
-  length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm);
+  length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm);
 
   tstrncpy(buf, ts, bufLen);
 }

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -179,6 +179,16 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
       } else {
         dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg);
         taosWriteQitem(pVnode->pWriteQ, pMsg);
+#if 0  // tests for batch writes
+        if (pMsg->msgType == TDMT_VND_CREATE_TABLE) {
+          SRpcMsg *pDup = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
+          memcpy(pDup, pMsg, sizeof(SRpcMsg));
+          pDup->pCont = rpcMallocCont(pMsg->contLen);
+          memcpy(pDup->pCont, pMsg->pCont, pMsg->contLen);
+          pDup->info.handle = NULL;
+          taosWriteQitem(pVnode->pWriteQ, pDup);
+        }
+#endif
       }
       break;
     case SYNC_QUEUE:

source/dnode/vnode/src/inc/sma.h

@@ -43,9 +43,9 @@ typedef struct SRSmaInfo     SRSmaInfo;
 typedef struct SRSmaInfoItem SRSmaInfoItem;
 
 struct SSmaEnv {
-  SRWLatch       lock;
-  int8_t         type;
-  SSmaStat      *pStat;
+  SRWLatch  lock;
+  int8_t    type;
+  SSmaStat *pStat;
 };
 
 typedef struct {
@@ -103,10 +103,15 @@ struct SRSmaInfoItem {
 };
 
 struct SRSmaInfo {
-  STSchema     *pTSchema;
-  int64_t       suid;
+  STSchema *pTSchema;
+  int64_t   suid;
+  int8_t    delFlag;
+  T_REF_DECLARE()
   SRSmaInfoItem items[TSDB_RETENTION_L2];
 };
+#define RSMA_INFO_HEAD_LEN   24
+#define RSMA_INFO_IS_DEL(r)  ((r)->delFlag == 1)
+#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1)
 
 enum {
   TASK_TRIGGER_STAT_INIT = 0,
@@ -120,8 +125,8 @@ enum {
 enum {
   RSMA_ROLE_CREATE = 0,
   RSMA_ROLE_DROP = 1,
-  RSMA_ROLE_FETCH = 2,
-  RSMA_ROLE_SUBMIT = 3,
+  RSMA_ROLE_SUBMIT = 2,
+  RSMA_ROLE_FETCH = 3,
   RSMA_ROLE_ITERATE = 4,
 };
 
@@ -134,6 +139,8 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg);
 
 int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat);
 int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat);
+int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo);
+int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo);
 
 void   *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
 int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
@@ -193,6 +200,7 @@ void           tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t le
 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree);
+void           tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid);
 int32_t        tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash);
 
 int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
@@ -258,8 +266,9 @@ void    tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
 void    tdCloseTFile(STFile *pTFile);
 void    tdDestroyTFile(STFile *pTFile);
 
-void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName);
-void tdGetVndDirName(int32_t vgId,const char *pdname,  const char *dname, bool endWithSep, char *outputName);
+void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version,
+                      char *outputName);
+void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/meta/metaTable.c

@@ -277,7 +277,7 @@ _drop_super_table:
 _exit:
   tdbFree(pKey);
   tdbFree(pData);
-  metaDebug("vgId:%d,  super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
+  metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
   return 0;
 }
 

source/dnode/vnode/src/sma/smaCommit.c

@@ -191,7 +191,7 @@ static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) {
   if ((pDir = taosOpenDir(dir)) == NULL) {
     regfree(&regex);
     terrno = TAOS_SYSTEM_ERROR(errno);
-    smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
+    smaDebug("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
     return TSDB_CODE_FAILED;
   }
 
@@ -392,5 +392,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
   // step 2: cleanup outdated qtaskinfo files
   tdCleanupQTaskInfoFiles(pSma, pRSmaStat);
 
+  atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0);
+
   return TSDB_CODE_SUCCESS;
 }

source/dnode/vnode/src/sma/smaEnv.c

@@ -169,6 +169,26 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
   return 0;
 }
 
+int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) {
+  if (!pRSmaInfo) return 0;
+
+  int ref = T_REF_INC(pRSmaInfo);
+  smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref);
+  return 0;
+}
+
+int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) {
+  if (!pRSmaInfo) return 0;
+
+  int ref = T_REF_DEC(pRSmaInfo);
+  smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref);
+
+  if (ref == 0) {
+    tdRemoveRSmaInfoBySuid(pSma, pRSmaInfo->suid);
+  }
+  return 0;
+}
+
 static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) {
   ASSERT(pSmaStat != NULL);
 

source/dnode/vnode/src/sma/smaRollup.c

@@ -24,6 +24,7 @@ SSmaMgmt smaMgmt = {
 };
 
 #define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver"
+#define TD_RSMAINFO_DEL_FILE      "rsmainfo.del"
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
 
@@ -48,14 +49,11 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables);
 static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed);
 static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed);
 
-
-
 static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) {
   // adapt accordingly if definition of SRSmaInfo update
   SRSmaInfo *pResult = NULL;
-  int32_t    rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *);
   ASSERT(pItem->level == TSDB_RETENTION_L1 || pItem->level == TSDB_RETENTION_L2);
-  pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + rsmaInfoHeadLen));
+  pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + RSMA_INFO_HEAD_LEN));
   ASSERT(pResult->pTSchema->numOfCols > 1);
   return pResult;
 }
@@ -116,8 +114,8 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
       if (pItem->taskInfo) {
         if (isDeepFree && pItem->tmrId) {
-          smaDebug("vgId:%d, table %" PRIi64 " stop fetch timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId,
-                   i + 1);
+          smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid,
+                   pItem->tmrId, i + 1);
           taosTmrStopA(&pItem->tmrId);
         }
         tdFreeQTaskInfo(&pItem->taskInfo, SMA_VID(pSma), i + 1);
@@ -337,6 +335,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
   }
   pRSmaInfo->pTSchema = pTSchema;
   pRSmaInfo->suid = suid;
+  T_REF_INIT_VAL(pRSmaInfo, 1);
 
   if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) {
     goto _err;
@@ -392,11 +391,33 @@ int32_t tdProcessRSmaCreate(SSma *pSma, SVCreateStbReq *pReq) {
 int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) {
   SVnode *pVnode = pSma->pVnode;
   if (!VND_IS_RSMA(pVnode)) {
-    smaTrace("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name,
+    smaTrace("vgId:%d, not drop rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name,
              pReq->suid);
     return TSDB_CODE_SUCCESS;
   }
 
+  SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
+  if (!pSmaEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+
+  SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pReq->suid);
+
+  if (!pRSmaInfo) {
+    smaWarn("vgId:%d, drop rsma for stable %s %" PRIi64 " failed no rsma in hash", TD_VID(pVnode), pReq->name,
+            pReq->suid);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // set del flag for data in mem
+  RSMA_INFO_SET_DEL(pRSmaInfo);
+  tdUnRefRSmaInfo(pSma, pRSmaInfo);
+
+  // save to file
+
   smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid);
   return TSDB_CODE_SUCCESS;
 }
@@ -650,10 +671,10 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
 
 /**
  * @brief During async commit, the SRSmaInfo object would be COW from iRSmaInfoHash and write lock should be applied.
- * 
- * @param pSma 
- * @param suid 
- * @return SRSmaInfo* 
+ *
+ * @param pSma
+ * @param suid
+ * @return SRSmaInfo*
  */
 static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
   SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
@@ -661,7 +682,6 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
   SRSmaInfo *pRSmaInfo = NULL;
 
   if (!pEnv) {
-    // only applicable when rsma env exists
     return NULL;
   }
 
@@ -683,18 +703,21 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
   SRSmaInfo *pCowRSmaInfo = NULL;
   // lock
   taosWLockLatch(SMA_ENV_LOCK(pEnv));
-  void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
-  if (iRSmaInfo) {
-    SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo;
-    if (pIRSmaInfo) {
-      if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) {
-        taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
-        smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr());
-        return NULL;
-      }
-      if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) {
-        taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
-        return NULL;
+  if (!taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t))) {  // 2-phase lock
+    void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+    if (iRSmaInfo) {
+      SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo;
+      if (pIRSmaInfo) {
+        if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) {
+          taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
+          smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr());
+          return NULL;
+        }
+        smaDebug("vgId:%d, clone rsma info succeed for suid:%" PRIu64, SMA_VID(pSma), suid);
+        if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) {
+          taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
+          return NULL;
+        }
       }
     }
   }
@@ -703,21 +726,56 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
   return pCowRSmaInfo;
 }
 
+/**
+ * @brief During the drop procedure, only need to delete the object in rsmaInfoHash.
+ *
+ * @param pSma
+ * @param suid
+ * @return SRSmaInfo*
+ */
+void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid) {
+  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
+  SRSmaStat *pStat = NULL;
+  SRSmaInfo *pRSmaInfo = NULL;
+
+  if (!pEnv) {
+    return;
+  }
+
+  pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
+  if (!pStat || !RSMA_INFO_HASH(pStat)) {
+    return;
+  }
+
+  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+  if (pRSmaInfo) {
+    if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+      tdFreeRSmaInfo(pSma, pRSmaInfo, true);
+    }
+    taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+    smaDebug("vgId:%d, remove from infoHash for table:%" PRIu64 " succeed", SMA_VID(pSma), suid);
+  }
+}
+
 static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
   SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid);
   if (!pRSmaInfo) {
-    smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
+    smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
     return TSDB_CODE_SUCCESS;
   }
 
   if (!pRSmaInfo->items[0].taskInfo) {
-    smaDebug("vgId:%d, return as no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid);
+    smaDebug("vgId:%d, execute rsma, no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid);
     return TSDB_CODE_SUCCESS;
   }
 
   if (inputType == STREAM_INPUT__DATA_SUBMIT) {
+    tdRefRSmaInfo(pSma, pRSmaInfo);
+
     tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1);
     tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2);
+
+    tdUnRefRSmaInfo(pSma, pRSmaInfo);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -931,10 +989,10 @@ _err:
 
 /**
  * @brief Restore from SRSmaQTaskInfoItem
- * 
- * @param pSma 
- * @param pItem 
- * @return int32_t 
+ *
+ * @param pSma
+ * @param pItem
+ * @return int32_t
  */
 static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) {
   SRSmaInfo *pRSmaInfo = NULL;
@@ -1271,6 +1329,9 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
   }
 
   SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem);
+  if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
+    goto _end;
+  }
 
   int8_t fetchTriggerStat =
       atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
@@ -1279,13 +1340,14 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
       smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
                pItem->level, pRSmaInfo->suid);
 
-      tdRefSmaStat(pSma, (SSmaStat *)pStat);
+      // sync procedure => async process
+      tdRefRSmaInfo(pSma, pRSmaInfo);
 
       SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
       qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
       tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK);
 
-      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
+      tdUnRefRSmaInfo(pSma, pRSmaInfo);
     } break;
     case TASK_TRIGGER_STAT_PAUSED: {
       smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",

source/dnode/vnode/src/tq/tqExec.c

@@ -68,7 +68,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa
       pRsp->rspOffset = *pOffset;
       return 0;
     } else {
-      tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1);
+      tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
       if (qStreamPrepareScan(task, pOffset) < 0) {
         pRsp->rspOffset = *pOffset;
         return 0;
@@ -106,7 +106,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa
     }
 
     if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
-      tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1);
+      tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
       qStreamPrepareScan(task, pOffset);
       continue;
     }

source/dnode/vnode/src/tq/tqPush.c

@@ -243,7 +243,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
     void* data = taosMemoryMalloc(msgLen);
     if (data == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      qError("failed to copy data for stream since out of memory");
+      tqError("failed to copy data for stream since out of memory");
       return -1;
     }
     memcpy(data, msg, msgLen);

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -53,6 +53,7 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
         *(int64_t *)(dc.data + dc.pos) = uid;
         *(int64_t *)(dc.data + dc.pos + 8) = ctime;
 
+        vTrace("vgId:%d, table:%s uid:%" PRId64 " is generated", pVnode->config.vgId, name, uid);
         tEndDecode(&dc);
       }
 
@@ -381,7 +382,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
     goto end;
   }
 
-  vInfo("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp);
+  vDebug("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp);
   int32_t ret = metaTtlDropTable(pVnode->pMeta, ttlReq.timestamp, tbUids);
   if (ret != 0) {
     goto end;

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -17,35 +17,22 @@
 #include "vnd.h"
 
 static inline bool vnodeIsMsgBlock(tmsg_t type) {
-  return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_ALTER_REPLICA);
+  return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) ||
+         (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL);
 }
 
 static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; }
 
-static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) {
-  if (!vnodeIsMsgBlock(type)) return;
-
-  int32_t count = atomic_add_fetch_32(&pVnode->blockCount, 1);
-  vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
-}
-
-static inline void vnodeWaitBlockMsg(SVnode *pVnode) {
-  int32_t count = atomic_load_32(&pVnode->blockCount);
-  if (count <= 0) return;
-
-  vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count);
-  tsem_wait(&pVnode->syncSem);
+static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
+  if (vnodeIsMsgBlock(pMsg->msgType)) {
+    vTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType));
+    tsem_wait(&pVnode->syncSem);
+  }
 }
 
-static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) {
-  if (!vnodeIsMsgBlock(type)) return;
-
-  int32_t count = atomic_load_32(&pVnode->blockCount);
-  if (count <= 0) return;
-
-  count = atomic_sub_fetch_32(&pVnode->blockCount, 1);
-  vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
-  if (count <= 0) {
+static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
+  if (vnodeIsMsgBlock(pMsg->msgType)) {
+    vTrace("vgId:%d, msg:%p post block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType));
     tsem_post(&pVnode->syncSem);
   }
 }
@@ -143,6 +130,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
   int32_t  code = 0;
   SRpcMsg *pMsg = NULL;
 
+  vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs);
+
   for (int32_t m = 0; m < numOfMsgs; m++) {
     if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
     const STraceId *trace = &pMsg->info.traceId;
@@ -165,13 +154,14 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
           if (rsp.info.handle != NULL) {
             tmsgSendRsp(&rsp);
           }
+        } else if (code == 0) {
+          vnodeWaitBlockMsg(pVnode, pMsg);
+        } else {
         }
       }
     }
 
-    if (code == 0) {
-      vnodeAccumBlockMsg(pVnode, pMsg->msgType);
-    } else if (code < 0) {
+    if (code < 0) {
       if (terrno == TSDB_CODE_SYN_NOT_LEADER) {
         vnodeRedirectRpcMsg(pVnode, pMsg);
       } else {
@@ -182,15 +172,12 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
           tmsgSendRsp(&rsp);
         }
       }
-    } else {
     }
 
     vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
     rpcFreeCont(pMsg->pCont);
     taosFreeQitem(pMsg);
   }
-
-  vnodeWaitBlockMsg(pVnode);
 }
 
 void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
@@ -213,7 +200,7 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
       }
     }
 
-    vnodePostBlockMsg(pVnode, pMsg->msgType);
+    vnodePostBlockMsg(pVnode, pMsg);
     if (rsp.info.handle != NULL) {
       tmsgSendRsp(&rsp);
     }
@@ -418,7 +405,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon
     tmsgSendRsp(&rpcMsg);
   }
 
-  vnodePostBlockMsg(pVnode, TDMT_VND_ALTER_REPLICA);
+  vnodePostBlockMsg(pVnode, pMsg);
 }
 
 static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {

source/libs/executor/src/timewindowoperator.c

@@ -63,7 +63,6 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo
                                       SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
                                       int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
                                       SExecTaskInfo* pTaskInfo) {
-  assert(win->skey <= win->ekey);
   SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                   masterscan, tableGroupId, pTaskInfo, true, pAggSup);
 

source/libs/function/src/builtins.c

@@ -194,8 +194,9 @@ static bool validateTimezoneFormat(const SValueNode* pVal) {
 void static addTimezoneParam(SNodeList* pList) {
   char       buf[6] = {0};
   time_t     t = taosTime(NULL);
-  struct tm* tmInfo = taosLocalTime(&t, NULL);
-  strftime(buf, sizeof(buf), "%z", tmInfo);
+  struct tm  tmInfo;
+  taosLocalTime(&t, &tmInfo);
+  strftime(buf, sizeof(buf), "%z", &tmInfo);
   int32_t len = (int32_t)strlen(buf);
 
   SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
@@ -1422,14 +1423,6 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
 }
 
 static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
-  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
-  for (int32_t i = 0; i < numOfParams; ++i) {
-    SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
-    if (QUERY_NODE_VALUE == nodeType(pParamNode)) {
-      return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
-    }
-  }
-
   pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType;
   return TSDB_CODE_SUCCESS;
 }

source/libs/nodes/src/nodesCodeFuncs.c

@@ -548,37 +548,12 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols";
 static const char* jkScanLogicPlanTableType = "TableType";
 static const char* jkScanLogicPlanTableId = "TableId";
 static const char* jkScanLogicPlanStableId = "StableId";
+static const char* jkScanLogicPlanScanType = "ScanType";
 static const char* jkScanLogicPlanScanCount = "ScanCount";
 static const char* jkScanLogicPlanReverseScanCount = "ReverseScanCount";
 static const char* jkScanLogicPlanTagCond = "TagCond";
 static const char* jkScanLogicPlanGroupTags = "GroupTags";
 
-// typedef struct SScanLogicNode {
-//   uint64_t      stableId;
-//   SVgroupsInfo* pVgroupList;
-//   EScanType     scanType;
-//   uint8_t       scanSeq[2];  // first is scan count, and second is reverse scan count
-//   STimeWindow   scanRange;
-//   SName         tableName;
-//   bool          showRewrite;
-//   double        ratio;
-//   SNodeList*    pDynamicScanFuncs;
-//   int32_t       dataRequired;
-//   int64_t       interval;
-//   int64_t       offset;
-//   int64_t       sliding;
-//   int8_t        intervalUnit;
-//   int8_t        slidingUnit;
-//   SNode*        pTagCond;
-//   SNode*        pTagIndexCond;
-//   int8_t        triggerType;
-//   int64_t       watermark;
-//   int8_t        igExpired;
-//   SArray*       pSmaIndexes;
-//   SNodeList*    pGroupTags;
-//   bool          groupSort;
-// } SScanLogicNode;
-
 static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
   const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
 
@@ -598,6 +573,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanStableId, pNode->stableId);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanType, pNode->scanType);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanCount, pNode->scanSeq[0]);
   }
@@ -634,6 +612,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetUBigIntValue(pJson, jkScanLogicPlanStableId, &pNode->stableId);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    tjsonGetNumberValue(pJson, jkScanLogicPlanScanType, pNode->scanType, code);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetUTinyIntValue(pJson, jkScanLogicPlanScanCount, &pNode->scanSeq[0]);
   }
@@ -1677,6 +1658,7 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) {
 }
 
 static const char* jkProjectPhysiPlanProjections = "Projections";
+static const char* jkProjectPhysiPlanMergeDataBlock = "MergeDataBlock";
 
 static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
   const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj;
@@ -1685,6 +1667,9 @@ static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkProjectPhysiPlanProjections, pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkProjectPhysiPlanMergeDataBlock, pNode->mergeDataBlock);
+  }
 
   return code;
 }
@@ -1696,6 +1681,9 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkProjectPhysiPlanProjections, &pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkProjectPhysiPlanMergeDataBlock, &pNode->mergeDataBlock);
+  }
 
   return code;
 }

source/libs/parser/src/parCalcConst.c

@@ -201,7 +201,8 @@ static int32_t calcConstProject(SNode* pProject, bool dual, SNode** pNew) {
 }
 
 static bool isUselessCol(SExprNode* pProj) {
-  if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId)) {
+  if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId) &&
+      !fmIsPseudoColumnFunc(((SFunctionNode*)pProj)->funcId)) {
     return false;
   }
   return NULL == ((SExprNode*)pProj)->pAssociation;

source/libs/parser/src/parTranslater.c

@@ -1686,7 +1686,7 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) {
 static bool sysTableFromVnode(const char* pTable) {
   return (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLES)) ||
          (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) ||
-         (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS)));
+          (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS)));
 }
 
 static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); }
@@ -5968,6 +5968,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
   if (NULL == pReq->tagName) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pReq->colId = pSchema->colId;
 
   SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema);
   if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) {
@@ -6051,6 +6052,7 @@ static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt,
   if (NULL == pReq->colName) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pReq->colId = pSchema->colId;
 
   return TSDB_CODE_SUCCESS;
 }
@@ -6071,6 +6073,7 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt
   if (NULL == pReq->colName) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pReq->colId = pSchema->colId;
 
   return TSDB_CODE_SUCCESS;
 }

source/libs/planner/src/planLogicCreater.c

@@ -20,6 +20,7 @@
 typedef struct SLogicPlanContext {
   SPlanContext* pPlanCxt;
   SLogicNode*   pCurrRoot;
+  bool          hasScan;
 } SLogicPlanContext;
 
 typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, void*, SLogicNode**);
@@ -161,6 +162,10 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols
     return SCAN_TYPE_STREAM;
   }
 
+  if (TSDB_SYSTEM_TABLE == tableType) {
+    return SCAN_TYPE_SYSTEM_TABLE;
+  }
+
   if (NULL == pScanCols) {
     return NULL == pScanPseudoCols
                ? SCAN_TYPE_TABLE
@@ -169,17 +174,6 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols
                       : SCAN_TYPE_TABLE);
   }
 
-  if (TSDB_SYSTEM_TABLE == tableType) {
-    return SCAN_TYPE_SYSTEM_TABLE;
-  }
-
-  SNode* pCol = NULL;
-  FOREACH(pCol, pScanCols) {
-    if (COLUMN_TYPE_COLUMN == ((SColumnNode*)pCol)->colType) {
-      return SCAN_TYPE_TABLE;
-    }
-  }
-
   return SCAN_TYPE_TABLE;
 }
 
@@ -300,6 +294,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
     nodesDestroyNode((SNode*)pScan);
   }
 
+  pCxt->hasScan = true;
+
   return code;
 }
 
@@ -1339,9 +1335,9 @@ static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) {
 
 static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); }
 
-static void setLogicSubplanType(SLogicSubplan* pSubplan) {
+static void setLogicSubplanType(bool hasScan, SLogicSubplan* pSubplan) {
   if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY != nodeType(pSubplan->pNode)) {
-    pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
+    pSubplan->subplanType = hasScan ? SUBPLAN_TYPE_SCAN : SUBPLAN_TYPE_MERGE;
   } else {
     SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)pSubplan->pNode;
     pSubplan->subplanType = (MODIFY_TABLE_TYPE_INSERT == pModify->modifyType && NULL != pModify->node.pChildren)
@@ -1351,7 +1347,7 @@ static void setLogicSubplanType(SLogicSubplan* pSubplan) {
 }
 
 int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) {
-  SLogicPlanContext cxt = {.pPlanCxt = pCxt};
+  SLogicPlanContext cxt = {.pPlanCxt = pCxt, .pCurrRoot = NULL, .hasScan = false};
 
   SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
   if (NULL == pSubplan) {
@@ -1364,7 +1360,7 @@ int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) {
   int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, &pSubplan->pNode);
   if (TSDB_CODE_SUCCESS == code) {
     setLogicNodeParent(pSubplan->pNode);
-    setLogicSubplanType(pSubplan);
+    setLogicSubplanType(cxt.hasScan, pSubplan);
   }
 
   if (TSDB_CODE_SUCCESS == code) {

source/libs/planner/src/planPhysiCreater.c

@@ -35,7 +35,8 @@ typedef struct SPhysiPlanContext {
   int32_t       errCode;
   int16_t       nextDataBlockId;
   SArray*       pLocationHelper;
-  SArray*       pExecNodeList;  // SArray<SQueryNodeLoad>
+  bool          hasScan;
+  bool          hasSysScan;
 } SPhysiPlanContext;
 
 static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
@@ -495,8 +496,6 @@ static int32_t createSimpleScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSub
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
-  SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0};
-  taosArrayPush(pCxt->pExecNodeList, &node);
   return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode);
 }
 
@@ -577,8 +576,6 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
     pSubplan->execNode.nodeId = MNODE_HANDLE;
     pSubplan->execNode.epSet = pCxt->pPlanCxt->mgmtEpSet;
   }
-  SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0};
-  taosArrayPush(pCxt->pExecNodeList, &node);
   if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_DNODE_VARIABLES)) {
     pScan->mgmtEpSet = pScanLogicNode->pVgroupList->vgroups->epSet;
   } else {
@@ -586,6 +583,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
   }
   tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName);
 
+  pCxt->hasSysScan = true;
   return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
 }
 
@@ -601,6 +599,7 @@ static int32_t createTableMergeScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
 
 static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
                                    SPhysiNode** pPhyNode) {
+  pCxt->hasScan = true;
   switch (pScanLogicNode->scanType) {
     case SCAN_TYPE_TAG:
     case SCAN_TYPE_BLOCK_INFO:
@@ -1806,23 +1805,31 @@ static void setExplainInfo(SPlanContext* pCxt, SQueryPlan* pPlan) {
   }
 }
 
+static void setExecNodeList(SPhysiPlanContext* pCxt, SArray* pExecNodeList) {
+  if (NULL == pExecNodeList) {
+    return;
+  }
+  if (pCxt->hasSysScan || !pCxt->hasScan) {
+    SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0};
+    taosArrayPush(pExecNodeList, &node);
+  }
+}
+
 int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) {
   SPhysiPlanContext cxt = {.pPlanCxt = pCxt,
                            .errCode = TSDB_CODE_SUCCESS,
                            .nextDataBlockId = 0,
                            .pLocationHelper = taosArrayInit(32, POINTER_BYTES),
-                           .pExecNodeList = pExecNodeList};
+                           .hasScan = false,
+                           .hasSysScan = false};
   if (NULL == cxt.pLocationHelper) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  if (QUERY_POLICY_VNODE == tsQueryPolicy) {
-    taosArrayClear(pExecNodeList);
-  }
-
   int32_t code = doCreatePhysiPlan(&cxt, pLogicPlan, pPlan);
   if (TSDB_CODE_SUCCESS == code) {
     setExplainInfo(pCxt, *pPlan);
+    setExecNodeList(&cxt, pExecNodeList);
   }
 
   destoryPhysiPlanContext(&cxt);

source/libs/scalar/src/sclfunc.c

@@ -1062,8 +1062,9 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *
       memmove(fraction, fraction + TSDB_TIME_PRECISION_SEC_DIGITS, TSDB_TIME_PRECISION_SEC_DIGITS);
     }
 
-    struct tm *tmInfo = taosLocalTime((const time_t *)&timeVal, NULL);
-    strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", tmInfo);
+    struct tm tmInfo;
+    taosLocalTime((const time_t *)&timeVal, &tmInfo);
+    strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo);
     int32_t len = (int32_t)strlen(buf);
 
     //add timezone string

source/libs/sync/inc/syncInt.h

@@ -192,6 +192,7 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms);
 int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode);
 int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode);
 int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode);
+int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode);
 
 // utils --------------
 int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);

source/libs/sync/src/syncMain.c

@@ -1298,6 +1298,12 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) {
   return ret;
 }
 
+int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) {
+  syncNodeStopHeartbeatTimer(pSyncNode);
+  syncNodeStartHeartbeatTimer(pSyncNode);
+  return 0;
+}
+
 // utils --------------
 int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
   SEpSet epSet;

source/libs/tdb/src/db/tdbBtree.c

@@ -103,10 +103,9 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
   // if pgno == 0 fetch new btree root leaf page
   if (pgno == 0) {
     // fetch page & insert into main db
-    // allocate a new child page
     SPage *pPage;
     TXN    txn;
-    tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0);
+    tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
 
     pPager->inTran = 1;
 
@@ -118,8 +117,6 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
       return -1;
     }
 
-    // TODO: Need to zero the page
-
     ret = tdbPagerWrite(pPager, pPage);
     if (ret < 0) {
       return -1;

source/libs/tdb/src/db/tdbPager.c

@@ -473,12 +473,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
     return -1;
   }
 
-  TXN txn;
-  tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-  SBtreeInitPageArg iArg;
-  iArg.pBt = pBt;
-  iArg.flags = 0;
-
   for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) {
     // read pgno & the page from journal
     SPgno  pgno;
@@ -494,20 +488,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
       return -1;
     }
 
-    /*
-    ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &iArg, &txn);
-    if (ret < 0) {
-      return -1;
-    }
-
-    // write the page to db
-    ret = tdbPagerWritePageToDB(pPager, pPage);
-    if (ret < 0) {
-      return -1;
-    }
-
-    tdbPCacheRelease(pPager->pCache, pPage, &txn);
-    */
     i64 offset = pPager->pageSize * (pgno - 1);
     if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
       ASSERT(0);
@@ -523,8 +503,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
 
   tdbOsFSync(pPager->fd);
 
-  tdbTxnClose(&txn);
-
   tdbOsFree(pageBuf);
 
   tdbOsClose(jfd);

source/os/src/osTime.c

@@ -357,14 +357,88 @@ FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) {
 
 time_t taosTime(time_t *t) { return time(t); }
 
-time_t taosMktime(struct tm *timep) { return mktime(timep); }
+time_t taosMktime(struct tm *timep) {
+#ifdef WINDOWS
+  struct tm tm1 = {0};
+  LARGE_INTEGER        t;
+  FILETIME             f;
+  SYSTEMTIME           s;
+  FILETIME             ff;
+  SYSTEMTIME           ss;
+  LARGE_INTEGER        offset;
+
+  time_t    tt = 0;
+  localtime_s(&tm1, &tt);
+  ss.wYear = tm1.tm_year + 1900;
+  ss.wMonth = tm1.tm_mon + 1;
+  ss.wDay = tm1.tm_wday;
+  ss.wHour = tm1.tm_hour;
+  ss.wMinute = tm1.tm_min;
+  ss.wSecond = tm1.tm_sec;
+  ss.wMilliseconds = 0;
+  SystemTimeToFileTime(&ss, &ff);
+  offset.QuadPart = ff.dwHighDateTime;
+  offset.QuadPart <<= 32;
+  offset.QuadPart |= ff.dwLowDateTime;
+
+  s.wYear = timep->tm_year + 1900;
+  s.wMonth = timep->tm_mon + 1;
+  s.wDay = timep->tm_wday;
+  s.wHour = timep->tm_hour;
+  s.wMinute = timep->tm_min;
+  s.wSecond = timep->tm_sec;
+  s.wMilliseconds = 0;
+  SystemTimeToFileTime(&s, &f);
+  t.QuadPart = f.dwHighDateTime;
+  t.QuadPart <<= 32;
+  t.QuadPart |= f.dwLowDateTime;
+
+  t.QuadPart -= offset.QuadPart;
+ return (time_t)(t.QuadPart / 10000000);
+#else
+  return mktime(timep);
+#endif
+ }
 
 struct tm *taosLocalTime(const time_t *timep, struct tm *result) {
   if (result == NULL) {
     return localtime(timep);
   }
 #ifdef WINDOWS
-  localtime_s(result, timep);
+  if (*timep < 0) {
+    SYSTEMTIME    ss,s;
+    FILETIME      ff,f;
+    LARGE_INTEGER offset;
+    struct tm     tm1;
+    time_t        tt = 0;
+    localtime_s(&tm1, &tt);
+    ss.wYear = tm1.tm_year + 1900;
+    ss.wMonth = tm1.tm_mon + 1;
+    ss.wDay = tm1.tm_mday;
+    ss.wHour = tm1.tm_hour;
+    ss.wMinute = tm1.tm_min;
+    ss.wSecond = tm1.tm_sec;
+    ss.wMilliseconds = 0;
+    SystemTimeToFileTime(&ss, &ff);
+    offset.QuadPart = ff.dwHighDateTime;
+    offset.QuadPart <<= 32;
+    offset.QuadPart |= ff.dwLowDateTime;
+    offset.QuadPart += *timep * 10000000;
+    f.dwLowDateTime = offset.QuadPart & 0xffffffff;
+    f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff;
+    FileTimeToSystemTime(&f, &s);
+    result->tm_sec = s.wSecond;
+    result->tm_min = s.wMinute;
+    result->tm_hour = s.wHour;
+    result->tm_mday = s.wDay;
+    result->tm_mon = s.wMonth-1;
+    result->tm_year = s.wYear-1900;
+    result->tm_wday = s.wDayOfWeek;
+    result->tm_yday = 0;
+    result->tm_isdst = 0;
+  } else {
+    localtime_s(result, timep);
+  }
 #else
   localtime_r(timep, result);
 #endif

tests/pytest/crash_gen/crash_gen_main.py

@@ -30,6 +30,7 @@ import argparse
 import sys
 import os
 import io
+import datetime
 import signal
 import traceback
 import requests
@@ -1107,14 +1108,20 @@ class Database:
     # TODO: fix the error as result of above: "tsdb timestamp is out of range"
     @classmethod
     def setupLastTick(cls):
-        t1 = datetime.datetime(2020, 6, 1)
+        # start time will be auto generated , start at 10 years ago  local time 
+        local_time = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-16]
+        local_epoch_time = [int(i) for i in local_time.split("-")]
+        #local_epoch_time will be such as : [2022, 7, 18]
+
+        t1 = datetime.datetime(local_epoch_time[0]-5, local_epoch_time[1], local_epoch_time[2])
         t2 = datetime.datetime.now()
         # maybe a very large number, takes 69 years to exceed Python int range
         elSec = int(t2.timestamp() - t1.timestamp())
         elSec2 = (elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500)) * \
             500  # a number representing seconds within 10 years
         # print("elSec = {}".format(elSec))
-        t3 = datetime.datetime(2012, 1, 1)  # default "keep" is 10 years
+
+        t3 = datetime.datetime(local_epoch_time[0]-10, local_epoch_time[1], local_epoch_time[2])  # default "keep" is 10 years
         t4 = datetime.datetime.fromtimestamp(
             t3.timestamp() + elSec2)  # see explanation above
         Logging.debug("Setting up TICKS to start from: {}".format(t4))

tests/script/jenkins/basic.txt

@@ -312,6 +312,7 @@
 ./test.sh -f tsim/valgrind/checkError3.sim
 ./test.sh -f tsim/valgrind/checkError4.sim
 ./test.sh -f tsim/valgrind/checkError5.sim
+./test.sh -f tsim/valgrind/checkError6.sim
 
 # --- vnode
 # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim

tests/script/tsim/db/alter_replica_31.sim

@@ -111,6 +111,15 @@ if $hasleader != 1 then
   goto step2
 endi
 
+# sql use db;
+# sql create table stb (ts timestamp, c int) tags (t int);
+# sql create table t0 using stb tags (0);
+# sql insert into t0 values(now, 1);
+# sql show db.stables;
+# sql show db.tables;
+# sql show db.vgroups;
+return
+
 sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd"
 sql create table db.ctb using db.stb tags(101, "102")
 sql insert into db.ctb values(now, 1, "2")

tests/script/tsim/valgrind/checkError5.sim

@@ -26,6 +26,7 @@ print =============== step2: create db
 sql create database db
 sql use db
 sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd"
+sql create table db.c1 using db.stb tags(101, 102, "103")
 
 print =============== step3: alter stb
 sql_error alter table db.stb add column ts int
@@ -42,9 +43,8 @@ sql alter table db.stb drop tag c1
 sql alter table db.stb drop tag t5
 sql alter table db.stb MODIFY tag t3 binary(32)
 sql alter table db.stb rename tag t1 tx
-
 sql alter table db.stb comment 'abcde' ;
-goto _OVER
+sql drop table db.stb
 
 print =============== step4: alter tb
 sql create table tb (ts timestamp, a int)
@@ -66,6 +66,35 @@ sql alter table tb add column h binary(10)
 sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb
 sql select * from tb order by ts desc
 
+print =============== step5: alter stb and insert data
+sql create table stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd"
+sql show db.stables
+sql describe stb
+sql_error alter table stb add column ts int
+
+sql create table db.ctb using db.stb tags(101, 102, "103")
+sql insert into db.ctb values(now, 1, "2")
+sql show db.tables
+sql select * from db.stb
+sql select * from tb
+
+sql alter table stb add column c3 int
+sql describe stb
+sql select * from db.stb
+sql select * from tb
+sql insert into db.ctb values(now+1s, 1, 2, 3)
+sql select * from db.stb
+
+sql alter table db.stb add column c4 bigint
+sql select * from db.stb
+sql insert into db.ctb values(now+2s, 1, 2, 3, 4)
+
+sql alter table db.stb drop column c1
+sql reset query cache
+sql select * from tb
+sql insert into db.ctb values(now+3s, 2, 3, 4)
+sql select * from db.stb
+
 _OVER:
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 print =============== check

tests/script/tsim/valgrind/checkError6.sim

@@ -68,7 +68,7 @@ $null=
 
 system_content sh/checkValgrind.sh -n dnode1 
 print cmd return result ----> [ $system_content ]
-if $system_content > 0 then
+if $system_content > 3 then
   return -1
 endi 
 

tests/system-test/0-others/cachelast.py → tests/system-test/0-others/cachemodel.py

@@ -56,6 +56,16 @@ class TDTestCase:
         }
         return numbers.get(value, 'other')
 
+    def getCacheModelNum(self,str):
+        numbers = {
+            "none" : 0,
+            "last_row" : 1,
+            "last_value" : 2,
+            "both" : 3
+
+        }
+        return numbers.get(str, 'other')
+
     def prepare_datas(self):
         for i in range(4):
             str = self.getCacheModelStr(i)
@@ -69,7 +79,7 @@ class TDTestCase:
                 tdSql.execute(" insert into tb1 values(now , %d, %f)" %(k,k*10) )
                 tdSql.execute(" insert into tb2 values(now , %d, %f)" %(k,k*10) )
 
-    def check_cache_last_sets(self):
+    def check_cachemodel_sets(self):
         
         
         # check cache_last value for database 
@@ -84,52 +94,54 @@ class TDTestCase:
             # print(cache_last_value)
             if dbname in ["information_schema" , "performance_schema"]:
                 continue
-            cache_lasts[dbname]=cache_last_value
+            cache_lasts[dbname]=self.getCacheModelNum(cache_last_value)
 
         
         # cache_last_set value 
         for k , v in cache_lasts.items():
             
-            if k=="testdb_"+str(v):
-                tdLog.info(" database %s cache_last value check pass, value is %s "%(k,v) )
+            if k=="testdb_"+str(self.getCacheModelStr(v)):
+                tdLog.info(" database %s cache_last value check pass, value is %s "%(k,self.getCacheModelStr(v)) )
             else:
-                tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,v) )
+                tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,self.getCacheModelStr(v)) )
 
         # # check storage layer implementation
 
 
-        # buildPath = self.getBuildPath()
-        # if (buildPath == ""):
-        #     tdLog.exit("taosd not found!")
-        # else:
-        #     tdLog.info("taosd found in %s" % buildPath)
-        # dataPath = buildPath + "/../sim/dnode1/data"
-        # abs_vnodePath = os.path.abspath(dataPath)+"/vnode/"
-        # tdLog.info("abs_vnodePath: %s" % abs_vnodePath)
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        dataPath = buildPath + "/../sim/dnode1/data"
+        abs_vnodePath = os.path.abspath(dataPath)+"/vnode/"
+        tdLog.info("abs_vnodePath: %s" % abs_vnodePath)
         
-        # tdSql.query(" show dnodes ")
-        # dnode_id  = tdSql.queryResult[0][0]
-
-        # for dbname in cache_lasts.keys():
-        #     print(dbname)
-        #     tdSql.execute(" use %s" % dbname)
-        #     tdSql.query(" show vgroups ")
-        #     vgroups_infos = tdSql.queryResult
-        #     for vgroup_info in vgroups_infos:
-        #         vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json"
-        #         vnode_info_of_db = f"cat {vnode_json}" 
-        #         vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8")
-        #         infoDict = json.loads(vnode_info)
-        #         vnode_json_of_dbname = f"{dnode_id}."+ dbname
-        #         config = infoDict["config"]
-        #         if infoDict["config"]["dbname"] == vnode_json_of_dbname:
-        #             if "cachelast" in infoDict["config"]:
-        #                 if int(infoDict["config"]["cachelast"]) != cache_lasts[dbname]:
-        #                     tdLog.exit("cachelast value is error in vnode.json of vnode%d "%(vgroup_info[0]))
-        #             else:
-        #                 tdLog.exit("cachelast not found in vnode.json of vnode%d "%(vgroup_info[0]))
-
-    def restart_check_cache_last_sets(self):
+        tdSql.query(" show dnodes ")
+        dnode_id  = tdSql.queryResult[0][0]
+
+        for dbname in cache_lasts.keys():
+            # print(dbname)
+            tdSql.execute(" use %s" % dbname)
+            tdSql.query(" show vgroups ")
+            vgroups_infos = tdSql.queryResult
+            for vgroup_info in vgroups_infos:
+                vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json"
+                vnode_info_of_db = f"cat {vnode_json}" 
+                vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8")
+                infoDict = json.loads(vnode_info)
+                vnode_json_of_dbname = f"{dnode_id}."+ dbname
+                config = infoDict["config"]
+                if infoDict["config"]["dbname"] == vnode_json_of_dbname:
+                    if "cacheLast" in infoDict["config"]:
+                        if int(infoDict["config"]["cacheLast"]) != cache_lasts[dbname]:
+                            tdLog.exit("cachemodel value is error in vnode.json of vnode%d "%(vgroup_info[0]))
+                        else:
+                            tdLog.info("cachemodel value is success in vnode.json of vnode%d "%(vgroup_info[0]))
+                    else:
+                        tdLog.exit("cacheLast not found in vnode.json of vnode%d "%(vgroup_info[0]))
+
+    def restart_check_cachemodel_sets(self):
         
         for i in range(3):
             tdSql.query("show dnodes")
@@ -137,14 +149,14 @@ class TDTestCase:
             tdDnodes.stop(index)
             tdDnodes.start(index)
             time.sleep(3)
-            self.check_cache_last_sets()
+            self.check_cachemodel_sets()
 
 
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         self.illegal_params()
         self.prepare_datas()
-        self.check_cache_last_sets()
-        self.restart_check_cache_last_sets()
+        self.check_cachemodel_sets()
+        self.restart_check_cachemodel_sets()
        
     def stop(self):
         tdSql.close()

tests/system-test/1-insert/delete_data.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import random
+import string
+
+from numpy import logspace
+from util import constant
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+from util.sqlset import TDSetSql
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(),logSql)
+        self.dbname = 'db_test'
+        self.setsql = TDSetSql()
+        self.ntbname = 'ntb'
+        self.rowNum = 10
+        self.tbnum = 20
+        self.ts = 1537146000000
+        self.binary_str = 'taosdata'
+        self.nchar_str = '涛思数据'
+        self.str_length = 20
+        self.column_dict = {
+            'col1': 'tinyint',
+            'col2': 'smallint',
+            'col3': 'int',
+            'col4': 'bigint',
+            'col5': 'tinyint unsigned',
+            'col6': 'smallint unsigned',
+            'col7': 'int unsigned',
+            'col8': 'bigint unsigned',
+            'col9': 'float',
+            'col10': 'double',
+            'col11': 'bool',
+            'col12': f'binary({self.str_length})',
+            'col13': f'nchar({self.str_length})',
+            
+        }
+        self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX)
+        self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX)
+        self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX)
+        self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX)
+        self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX)
+        self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX)
+        self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX)
+        self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX)
+        self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX)
+        self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300))
+        self.bool_val = random.randint(0,100)%2
+        self.binary_val = tdCom.getLongName(random.randint(0,self.str_length))
+        self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length))
+        self.base_data = {
+            'tinyint':self.tinyint_val,
+            'smallint':self.smallint_val,
+            'int':self.int_val,
+            'bigint':self.bigint_val,
+            'tinyint unsigned':self.untingint_val,
+            'smallint unsigned':self.unsmallint_val,
+            'int unsigned':self.unint_val,
+            'bigint unsigned':self.unbigint_val,
+            'bool':self.bool_val,
+            'float':self.float_val,
+            'double':self.double_val,
+            'binary':self.binary_val,
+            'nchar':self.nchar_val
+                    }
+    def insert_base_data(self,col_type,tbname,rows,base_data):
+        for i in range(rows):
+            if col_type.lower() == 'tinyint':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})')
+            elif col_type.lower() == 'smallint':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})')
+            elif col_type.lower() == 'int':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})')
+            elif col_type.lower() == 'bigint':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})')
+            elif col_type.lower() == 'tinyint unsigned':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})')
+            elif col_type.lower() == 'smallint unsigned':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})')
+            elif col_type.lower() == 'int unsigned':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})')
+            elif col_type.lower() == 'bigint unsigned':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})')
+            elif col_type.lower() == 'bool':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})')    
+            elif col_type.lower() == 'float':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})')      
+            elif col_type.lower() == 'double':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})')
+            elif 'binary' in col_type.lower():
+                tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''')
+            elif 'nchar' in col_type.lower():
+                tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''')        
+
+    def delete_all_data(self,tbname,col_type,row_num,base_data,dbname):
+        tdSql.execute(f'delete from {tbname}')
+        tdSql.execute(f'flush database {dbname}')
+        tdSql.execute('reset query cache')
+        tdSql.query(f'select * from {tbname}')
+        tdSql.checkRows(0)
+        self.insert_base_data(col_type,tbname,row_num,base_data)
+        tdSql.execute(f'flush database {dbname}')
+        tdSql.execute('reset query cache')
+        tdSql.query(f'select * from {tbname}')
+        tdSql.checkRows(row_num)
+    def delete_one_row(self,tbname,column_type,column_name,base_data,dbname):
+        tdSql.execute(f'delete from {tbname} where ts={self.ts}')
+        tdSql.execute(f'flush database {dbname}')
+        tdSql.execute('reset query cache')
+        tdSql.query(f'select {column_name} from {tbname}')
+        tdSql.checkRows(self.rowNum-1)
+        tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}')
+        tdSql.checkRows(0)
+        if 'binary' in column_type.lower():
+            tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''')
+        elif 'nchar' in column_type.lower():
+            tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''')
+        else:
+            tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})')
+        tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}')
+        if column_type.lower() == 'float' or column_type.lower() == 'double':
+            if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001:
+                tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0])
+            else:
+                tdLog.exit(f'{column_type} data check failure')
+        elif 'binary' in column_type.lower():
+            tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary'])
+        elif 'nchar' in column_type.lower():
+            tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar'])
+        else:
+            tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type])
+    
+    def delete_rows(self):
+
+
+        pass
+    def delete_error(self,tbname,column_name,column_type,base_data):
+        for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']:
+            if 'binary' in column_type.lower():
+                tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''')
+            elif 'nchar' in column_type.lower():
+                tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''')
+            else:
+                tdSql.error('delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}')
+                
+    def delete_data_ntb(self):
+        tdSql.execute(f'create database if not exists {self.dbname}')
+        tdSql.execute(f'use {self.dbname}')
+        for col_name,col_type in self.column_dict.items():
+            tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})')
+            self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data)
+            self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.dbname)
+            self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname)
+            self.delete_error(self.ntbname,col_name,col_type,self.base_data)
+            for i in range(self.rowNum):
+                tdSql.execute(f'delete from {self.ntbname} where ts>{self.ts+i}')
+                tdSql.execute(f'flush database {self.dbname}')
+                tdSql.execute('reset query cache')
+                tdSql.query(f'select {col_name} from {self.ntbname}')
+                tdSql.checkRows(i+1)
+                self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data)
+            
+            tdSql.execute(f'drop table {self.ntbname}')
+        
+    def run(self):
+        self.delete_data_ntb()
+        
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/system-test/1-insert/update_data.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import random
+import string
+from util import constant
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+from util.sqlset import TDSetSql
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(),logSql)
+        self.setsql = TDSetSql()
+        self.dbname = 'db_test'
+        self.ntbname = 'ntb'
+        self.stbname = 'stb'
+        self.ctbname = 'ctb'
+        self.ts = 1537146000000
+        self.str_length = 20
+        self.column_dict = {
+            'col1': 'tinyint',
+            'col2': 'smallint',
+            'col3': 'int',
+            'col4': 'bigint',
+            'col5': 'tinyint unsigned',
+            'col6': 'smallint unsigned',
+            'col7': 'int unsigned',
+            'col8': 'bigint unsigned',
+            'col9': 'float',
+            'col10': 'double',
+            'col11': 'bool',
+            'col12': f'binary({self.str_length})',
+            'col13': f'nchar({self.str_length})',
+            'col_ts'  : 'timestamp'
+        }
+           
+    def data_check(self,tbname,col_name,col_type,value):
+        tdSql.query(f'select {col_name} from {tbname}')
+        if col_type.lower() == 'float' or col_type.lower() == 'double':
+            if abs(tdSql.queryResult[0][0] - value) / value <= 0.0001:
+                tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0])
+            else:
+                tdLog.exit(f'{col_name} data check failure')
+        elif col_type.lower() == 'timestamp':
+            tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f")))
+        else:
+            tdSql.checkEqual(tdSql.queryResult[0][0],value)
+    def update_and_check_data(self,tbname,col_name,col_type,value,dbname):
+        if 'binary' in col_type.lower() or 'nchar' in col_type.lower():
+            tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")')
+        else:
+            tdSql.execute(f'insert into {tbname} values({self.ts},{value})')
+        self.data_check(tbname,col_name,col_type,value)
+        tdSql.execute(f'flush database {dbname}')
+        tdSql.execute('reset query cache')
+        self.data_check(tbname,col_name,col_type,value)
+        for func in ['first','last']:
+            tdSql.execute(f'select {func}({col_name}) from {tbname}')
+    def error_check(self,tbname,column_dict,tb_type=None,stbname=None):
+        str_length = self.str_length+1
+        for col_name,col_type in column_dict.items():
+            if tb_type == 'ntb':
+                tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})')
+            elif tb_type == 'ctb':
+                tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)')
+                tdSql.execute(f'create table {tbname} using {stbname} tags(1)')
+            tdSql.execute(f'insert into {tbname} values({self.ts},null)')
+            if col_type.lower() == 'double':
+                for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'float':
+                for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif 'binary' in col_type.lower() or 'nchar' in col_type.lower():
+                for error_value in [tdCom.getLongName(str_length)]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},"{error_value}")')
+            elif col_type.lower() == 'bool':
+                for error_value in [tdCom.getLongName(self.str_length)]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'tinyint':
+                for error_value in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'smallint':
+                for error_value in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'int':
+                for error_value in [constant.INT_MIN-1,constant.INT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'bigint':
+                for error_value in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'tinyint unsigned':
+                for error_value in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')     
+            elif col_type.lower() == 'smallint unsigned':
+                for error_value in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'int unsigned':
+                for error_value in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'bigint unsigned':
+                for error_value in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')           
+            tdSql.execute(f'drop table {tbname}')
+            if tb_type == 'ctb':
+                tdSql.execute(f'drop table {stbname}')
+    def update_data_check(self,tbname,column_dict,dbname,tb_type=None,stbname=None):
+        up_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX)
+        up_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX)
+        up_int = random.randint(constant.INT_MIN,constant.INT_MAX)
+        up_bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX)
+        up_untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX)
+        up_unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX)
+        up_unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX)
+        up_unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX)
+        up_bool = random.randint(0,100)%2
+        up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX)
+        up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300))
+        binary_length = random.randint(0,self.str_length)
+        nchar_length = random.randint(0,self.str_length)
+        up_binary = tdCom.getLongName(binary_length)
+        up_nchar = tdCom.getLongName(nchar_length)
+        for col_name,col_type in column_dict.items():
+            if tb_type == 'ntb':
+                tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})')
+            elif tb_type == 'ctb':
+                tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)')
+                tdSql.execute(f'create table {tbname} using {stbname} tags(1)')
+            tdSql.execute(f'insert into {tbname} values({self.ts},null)')
+            if col_type.lower() == 'tinyint':
+                self.update_and_check_data(tbname,col_name,col_type,up_tinyint,dbname)
+            elif col_type.lower() == 'smallint':
+                self.update_and_check_data(tbname,col_name,col_type,up_smallint,dbname)
+            elif col_type.lower() == 'int':
+                self.update_and_check_data(tbname,col_name,col_type,up_int,dbname)
+            elif col_type.lower() == 'bigint':
+                self.update_and_check_data(tbname,col_name,col_type,up_bigint,dbname)
+            elif col_type.lower() == 'tinyint unsigned':
+                self.update_and_check_data(tbname,col_name,col_type,up_untinyint,dbname)
+            elif col_type.lower() == 'smallint unsigned':
+                self.update_and_check_data(tbname,col_name,col_type,up_unsmallint,dbname)
+            elif col_type.lower() == 'int unsigned':
+                self.update_and_check_data(tbname,col_name,col_type,up_unint,dbname)
+            elif col_type.lower() == 'bigint unsigned':
+                self.update_and_check_data(tbname,col_name,col_type,up_unbigint,dbname)
+            elif col_type.lower() == 'bool':
+                self.update_and_check_data(tbname,col_name,col_type,up_bool,dbname)    
+            elif col_type.lower() == 'float':
+                self.update_and_check_data(tbname,col_name,col_type,up_float,dbname)      
+            elif col_type.lower() == 'double':
+                self.update_and_check_data(tbname,col_name,col_type,up_double,dbname)
+            elif 'binary' in col_type.lower():
+                self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname)
+            elif 'nchar' in col_type.lower():
+                self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname)
+            elif col_type.lower() == 'timestamp':
+                self.update_and_check_data(tbname,col_name,col_type,self.ts+1,dbname)
+            tdSql.execute(f'insert into {tbname} values({self.ts},null)')
+            tdSql.query(f'select {col_name} from {tbname}')
+            tdSql.checkEqual(tdSql.queryResult[0][0],None)
+            tdSql.execute(f'flush database {self.dbname}')
+            tdSql.execute('reset query cache')
+            tdSql.query(f'select {col_name} from {tbname}')
+            tdSql.checkEqual(tdSql.queryResult[0][0],None)
+            tdSql.execute(f'drop table {tbname}')
+            if tb_type == 'ctb':
+                tdSql.execute(f'drop table {stbname}')
+    def update_check(self):
+        tdSql.execute(f'drop database if exists {self.dbname}')
+        tdSql.execute(f'create database {self.dbname}')
+        tdSql.execute(f'use {self.dbname}')
+        self.update_data_check(self.ntbname,self.column_dict,self.dbname,'ntb')
+        for col_name,col_type in self.column_dict.items():
+            tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})')
+            tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)')
+            if 'binary' in col_type.lower():
+                up_binary = tdCom.getLongName(self.str_length+1)
+                tdSql.execute(f'alter table {self.ntbname} modify column {col_name} binary({self.str_length+1})')
+                self.update_and_check_data(self.ntbname,col_name,col_type,up_binary,self.dbname)
+            elif 'nchar' in col_type.lower():
+                up_nchar = tdCom.getLongName(self.str_length+1)
+                tdSql.execute(f'alter table {self.ntbname} modify column {col_name} nchar({self.str_length+1})')
+                self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname)
+            tdSql.execute(f'drop table {self.ntbname}')
+        self.update_data_check(self.ctbname,self.column_dict,self.dbname,'ctb',self.stbname)
+        for col_name,col_type in self.column_dict.items():
+            tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)')
+            tdSql.execute(f'create table {self.ctbname} using {self.stbname} tags(1)')
+            tdSql.execute(f'insert into {self.ctbname} values({self.ts},null)')
+            if 'binary' in col_type.lower():
+                up_binary = tdCom.getLongName(self.str_length+1)
+                tdSql.execute(f'alter table {self.stbname} modify column {col_name} binary({self.str_length+1})')
+                self.update_and_check_data(self.ctbname,col_name,col_type,up_binary,self.dbname)
+            elif 'nchar' in col_type.lower():
+                up_nchar = tdCom.getLongName(self.str_length+1)
+                tdSql.execute(f'alter table {self.stbname} modify column {col_name} nchar({self.str_length+1})')
+                self.update_and_check_data(self.ctbname,col_name,col_type,up_nchar,self.dbname)
+            tdSql.execute(f'drop table {self.stbname}')
+
+    def update_check_error(self):
+        tdSql.execute(f'drop database if exists {self.dbname}')
+        tdSql.execute(f'create database {self.dbname}')
+        tdSql.execute(f'use {self.dbname}')
+        self.error_check(self.ntbname,self.column_dict,'ntb')
+        self.error_check(self.ctbname,self.column_dict,'ctb',self.stbname)
+
+    def run(self):
+        self.update_check()
+        self.update_check_error()
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/system-test/2-query/last_row.py

@@ -48,7 +48,7 @@ class TDTestCase:
                 c9 = "'nchar_val'"
                 c10 = ts
                 tdSql.execute(f" insert into  {tbname} values ({ts},{c1},{c2},{c3},{c4},{c5},{c6},{c7},{c8},{c9},{c10})")
-        
+
         tdSql.execute("use test")
         tbnames = ["stb", "sub_tb_1"]
         support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"]
@@ -61,7 +61,7 @@ class TDTestCase:
                 origin_sql = "select {} from {} order by tbname".format(colname, 'test.'+tbname)
                 if coltype[1] in support_types:
                     self.check_result_auto(origin_sql , abs_sql)
-                    
+
     def prepare_datas(self ,cache_value):
         tdSql.execute("drop database if exists db ")
         create_db_sql = f"create database if not exists db keep 3650 duration 1000 cachemodel {cache_value}"
@@ -220,22 +220,22 @@ class TDTestCase:
     def test_errors(self):
         tdSql.execute("use testdb")
 
-        # bug need fix 
-        tdSql.error("select last_row(c1 ,NULL) from testdb.t1")
+        # bug need fix
+        tdSql.query("select last_row(c1 ,NULL) from testdb.t1")
 
         error_sql_lists = [
             "select last_row from testdb.t1",
             "select last_row(-+--+c1) from testdb.t1",
             "select last_row(123--123)==1 from testdb.t1",
             "select last_row(c1) as 'd1' from testdb.t1",
-            "select last_row(c1 ,NULL) from testdb.t1",
+            #"select last_row(c1 ,NULL) from testdb.t1",
             "select last_row(,) from testdb.t1;",
             "select last_row(abs(c1) ab from testdb.t1)",
             "select last_row(c1) as int from testdb.t1",
             "select last_row from testdb.stb1",
             "select last_row(123--123)==1 from testdb.stb1",
             "select last_row(c1) as 'd1' from testdb.stb1",
-            "select last_row(c1 ,NULL) from testdb.stb1",
+            #"select last_row(c1 ,NULL) from testdb.stb1",
             "select last_row(,) from testdb.stb1;",
             "select last_row(abs(c1) ab from testdb.stb1)",
             "select last_row(c1) as int from testdb.stb1"
@@ -246,7 +246,7 @@ class TDTestCase:
     def support_types(self):
         tdSql.execute("use testdb")
         tbnames = ["stb1", "t1", "ct1", "ct2"]
-        
+
         for tbname in tbnames:
             tdSql.query("desc {}".format(tbname))
             coltypes = tdSql.queryResult
@@ -256,7 +256,7 @@ class TDTestCase:
                 if col_note != "TAG":
                     abs_sql = "select last_row({}) from {}".format(colname, "testdb."+tbname)
                     tdSql.query(abs_sql)
-                
+
 
     def basic_abs_function(self):
 
@@ -283,7 +283,7 @@ class TDTestCase:
 
         # used for regular table
 
-        # bug need fix 
+        # bug need fix
         tdSql.query("select last_row(c1) from testdb.t1")
         tdSql.checkData(0, 0, None)
         tdSql.query("select last_row(c1) from testdb.ct4")
@@ -291,21 +291,21 @@ class TDTestCase:
         tdSql.query("select last_row(c1) from testdb.stb1")
         tdSql.checkData(0, 0, None)
 
-        # # bug need fix 
+        # # bug need fix
         tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.t1")
         tdSql.checkData(0, 0, None)
         tdSql.checkData(0, 1, None)
         tdSql.checkData(0, 2, None)
 
-        # # bug need fix 
+        # # bug need fix
         tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.ct1")
         tdSql.checkData(0, 0, 9)
         tdSql.checkData(0, 1, -99999)
         tdSql.checkData(0, 2, -999)
         tdSql.checkData(0, 3, None)
         tdSql.checkData(0, 4,-9.99000)
-        
-        # bug need fix 
+
+        # bug need fix
         tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.stb1 where tbname='ct1'")
         tdSql.checkData(0, 0, 9)
         tdSql.checkData(0, 1, -99999)
@@ -313,14 +313,14 @@ class TDTestCase:
         tdSql.checkData(0, 3, None)
         tdSql.checkData(0, 4,-9.99000)
 
-        # bug fix 
+        # bug fix
         tdSql.query("select last_row(abs(c1)) from testdb.ct1")
         tdSql.checkData(0,0,9)
 
-        # # bug fix 
+        # # bug fix
         tdSql.query("select last_row(c1+1) from testdb.ct1")
-        tdSql.query("select last_row(c1+1) from testdb.stb1") 
-        tdSql.query("select last_row(c1+1) from testdb.t1")      
+        tdSql.query("select last_row(c1+1) from testdb.stb1")
+        tdSql.query("select last_row(c1+1) from testdb.t1")
 
         # used for stable table
         tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from testdb.ct1")
@@ -329,7 +329,7 @@ class TDTestCase:
         tdSql.checkData(0,2,-999)
         tdSql.checkData(0,3,None)
 
-        # bug need fix 
+        # bug need fix
         tdSql.query("select last_row(c1 ,c2 ,c3) from testdb.stb1 ")
         tdSql.checkData(0,0,None)
         tdSql.checkData(0,1,None)
@@ -338,7 +338,7 @@ class TDTestCase:
 
         tdSql.query('select last_row(c1) from testdb.t1 where ts <"2022-12-31 01:01:36.000"')
         tdSql.checkData(0,0,8)
-        # bug need fix 
+        # bug need fix
         tdSql.query("select abs(last_row(c1)-2)+max(c1),ceil(last_row(c4)-2) from testdb.stb1 where c4 is not null")
         tdSql.checkData(0,0,16.000000000)
         tdSql.checkData(0,1,-101.000000000)
@@ -371,7 +371,7 @@ class TDTestCase:
         tdSql.query("select last_row(c1) ,count(*)  from testdb.stb1 where  c1 is null")
         tdSql.checkData(0,0,None)
         tdSql.checkData(0,1,3)
-        
+
         tdSql.query("select last_row(c1) ,count(c1)  from testdb.stb1 where  c1 is null")
         tdSql.checkData(0,0,None)
         tdSql.checkData(0,1,0)
@@ -380,7 +380,7 @@ class TDTestCase:
         tdSql.query("select tbname ,last_row(c1) from testdb.stb1")
         tdSql.checkData(0,0,'ct4')
         tdSql.checkData(0,1,None)
-        
+
         tdSql.query(" select tbname ,last_row(c1) from testdb.stb1 partition by tbname order by tbname ")
         tdSql.checkData(0,0,'ct1')
         tdSql.checkData(0,1,9)
@@ -396,11 +396,11 @@ class TDTestCase:
         tdSql.query(" select t1 ,count(c1) from testdb.stb1 partition by t1 ")
         tdSql.checkRows(2)
 
-        # filter by tbname 
+        # filter by tbname
         tdSql.query("select last_row(c1) from testdb.stb1 where tbname = 'ct1' ")
         tdSql.checkData(0,0,9)
 
-        # bug need fix 
+        # bug need fix
         tdSql.query("select tbname ,last_row(c1) from testdb.stb1 where tbname = 'ct1' ")
         tdSql.checkData(0,1,9)
         tdSql.query("select tbname ,last_row(c1) from testdb.stb1 partition by tbname order by tbname")
@@ -428,7 +428,7 @@ class TDTestCase:
         tdSql.checkData(0,2,333)
         tdSql.checkData(0,3,3)
 
-        # filter by tag 
+        # filter by tag
         tdSql.query("select tbname ,last_row(c1) from testdb.stb1 where t1 =0 ")
         tdSql.checkData(0,1,9)
         tdSql.query("select tbname ,last_row(c1) ,t1 from testdb.stb1 partition by t1 order by t1")
@@ -437,7 +437,7 @@ class TDTestCase:
         tdSql.checkData(1, 0, 'ct4')
         tdSql.checkData(1, 1, None)
 
-        # filter by col 
+        # filter by col
 
         tdSql.query("select tbname ,last_row(c1),abs(c1)from testdb.stb1 where c1 =1;")
         tdSql.checkData(0, 0, 'ct1')
@@ -445,7 +445,7 @@ class TDTestCase:
         tdSql.checkData(0, 2, 1)
         tdSql.query("select last_row(c1) from testdb.stb1 where abs(ceil(c1))*c1==1")
         tdSql.checkData(0,0,1)
-    
+
         # mix with common functions
         tdSql.query("select last_row(*) ,last(*) from testdb.stb1  ")
         tdSql.checkRows(1)
@@ -457,11 +457,11 @@ class TDTestCase:
         tdSql.query("select last_row(c1+abs(c1)) from testdb.stb1 partition by tbname order by tbname")
         tdSql.query("select last(c1), max(c1+abs(c1)),last_row(c1+abs(c1)) from testdb.stb1 partition by tbname order by tbname")
 
-        # # bug need fix ,taosd crash 
+        # # bug need fix ,taosd crash
         tdSql.error("select last_row(*) ,last(*) from testdb.stb1 partition by tbname order by last(*)")
         tdSql.error("select last_row(*) ,last(*) from testdb.stb1 partition by tbname order by last_row(*)")
 
-        # mix with agg functions 
+        # mix with agg functions
         tdSql.query("select last(*), last_row(*),last(c1), last_row(c1) from testdb.stb1 ")
         tdSql.query("select last(*), last_row(*),last(c1), last_row(c1) from testdb.ct1 ")
         tdSql.query("select last(*), last_row(*),last(c1+1)*max(c1), last_row(c1+2)/2 from testdb.t1 ")
@@ -564,7 +564,7 @@ class TDTestCase:
 
         # bug need fix
 
-        tdSql.query(" select sum(c1) from testdb.stb1 where t1+10 >1; ")  
+        tdSql.query(" select sum(c1) from testdb.stb1 where t1+10 >1; ")
         tdSql.query("select c1 ,t1 from testdb.stb1 where t1 =0 ")
         tdSql.checkRows(13)
         tdSql.query("select last_row(c1,t1) from testdb.stb1 ")
@@ -627,8 +627,8 @@ class TDTestCase:
         tdSql.execute(" use testdb ")
         tdSql.query(" select last_row(c1) from testdb.stb1 group by t1 order by t1 ")
         tdSql.checkRows(2)
-        
-        # bug need fix 
+
+        # bug need fix
         tdSql.query("select last_row(c1) from testdb.stb1 group by c1 order by c1,t1 ")
         tdSql.checkRows(10)
         tdSql.checkData(9,0,8)
@@ -643,7 +643,7 @@ class TDTestCase:
         tdSql.checkRows(11)
         tdSql.checkData(10,0,9)
 
-        # bug need fix , result is error 
+        # bug need fix , result is error
         tdSql.query("select last_row(c1) from testdb.ct4 group by c1 order by t1 ")
         tdSql.query("select last_row(t1) from testdb.ct4 group by c1 order by t1 ")
 
@@ -661,24 +661,24 @@ class TDTestCase:
         tdSql.query("select last_row(c1+c3) from testdb.stb1 group by abs(c1+c3) order by abs(c1+c3)")
         tdSql.checkRows(11)
 
-        # bug need fix , taosd crash 
+        # bug need fix , taosd crash
         tdSql.query("select last_row(c1+c3)+c2 from testdb.stb1 group by abs(c1+c3)+c2 order by abs(c1+c3)+c2")
         tdSql.checkRows(11)
         tdSql.query("select last_row(c1+c3)+last_row(c2) from testdb.stb1 group by abs(c1+c3)+abs(c2) order by abs(c1+c3)+abs(c2)")
         tdSql.checkRows(11)
         tdSql.checkData(0,0,None)
         tdSql.checkData(2,0,11223.000000000)
-        
+
         tdSql.query("select last_row(t1) from testdb.stb1 where abs(c1+t1)=1 partition by tbname")
         tdSql.checkData(0,0,1)
-        
+
         tdSql.query("select tbname , last_row(c1) from testdb.stb1 partition by tbname order by tbname")
         tdSql.checkRows(2)
         tdSql.checkData(0, 0, 'ct1')
         tdSql.checkData(0, 1,  9)
         tdSql.checkData(0, 2, 'ct4')
         tdSql.checkData(0, 3, None)
-        
+
         tdSql.query("select tbname , last_row(c1) from testdb.stb1 partition by t1 order by t1")
         tdSql.checkRows(2)
         tdSql.checkData(0, 0, 'ct1')
@@ -686,7 +686,7 @@ class TDTestCase:
         tdSql.checkData(0, 2, 'ct4')
         tdSql.checkData(0, 3, None)
 
-        # bug need fix 
+        # bug need fix
         tdSql.query("select tbname , last_row(c1) from testdb.stb1 partition by c2 order by c1")
         tdSql.checkRows(11)
         tdSql.checkData(10,1,9)
@@ -700,7 +700,7 @@ class TDTestCase:
 
         tdSql.query("select abs(c1) ,c2 ,t1, last_row(t1) from testdb.stb1 partition by c2 order by t1")
         tdSql.checkRows(11)
-   
+
         tdSql.query("select t1 ,last_row(t1) ,c2 from testdb.stb1 partition by c2 order by t1")
         tdSql.checkRows(11)
 
@@ -722,13 +722,14 @@ class TDTestCase:
         tdSql.query("select last_row(ceil(c1-2)) , abs(floor(t1+1)) ,floor(c2-c1) from testdb.stb1 partition by abs(floor(c1)) order by abs(c1)")
         tdSql.checkRows(11)
 
-        # interval 
 
         tdSql.query("select max(c1) from stb1 interval(50s) sliding(30s)")
         tdSql.checkRows(13)
 
         tdSql.query("select unique(c1) from stb1 partition by tbname")
 
+        # interval
+
         tdSql.query("select last_row(c1) from testdb.stb1 interval(50s) sliding(30s)")
         tdSql.checkRows(27)
 
@@ -752,13 +753,13 @@ class TDTestCase:
         tdSql.query('select last_row(c1) from testdb.stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)')
         tdSql.query('select last_row(c1) from (select ts ,  c1  from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" ) interval(10s) sliding(5s)')
 
-        # join 
+        # join
         tdSql.query("use test")
         tdSql.query("select last(sub_tb_1.c1), last(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts")
         tdSql.checkCols(2)
         last_row_result = tdSql.queryResult
         tdSql.query("select last_row(sub_tb_1.c1), last_row(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts")
-        
+
         for ind , row in enumerate(last_row_result):
             tdSql.checkData(ind , 0 , row[0])
 
@@ -776,7 +777,7 @@ class TDTestCase:
         tdSql.query("select last_row(*), last(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts")
         for ind , row in enumerate(last_row_result):
             tdSql.checkData(ind , 0 , row[0])
-        
+
 
     def support_super_table_test(self):
         tdSql.execute(" use testdb ")
@@ -790,9 +791,9 @@ class TDTestCase:
         self.check_result_auto( " select t3,c1 from testdb.stb1 where c1 > 0 order by tbname  " , "select t3 ,abs(c1) from testdb.stb1 where c1 > 0 order by tbname" )
         self.check_result_auto( " select t4,c1 from testdb.stb1 where c1 > 0 order by tbname  " , "select t4 , abs(c1) from testdb.stb1 where c1 > 0 order by tbname" )
         pass
-    
+
     def basic_query(self):
-        
+
         tdLog.printNoPrefix("==========step2:test errors ==============")
 
         self.test_errors()
@@ -835,19 +836,19 @@ class TDTestCase:
         self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'NONE'")
         self.basic_query()
 
-        # cache_last 1 
+        # cache_last 1
         self.prepare_datas("'LAST_ROW'")
         self.prepare_tag_datas("'LAST_ROW'")
         self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_ROW'")
         self.basic_query()
 
-        # cache_last 2 
+        # cache_last 2
         self.prepare_datas("'LAST_VALUE'")
         self.prepare_tag_datas("'LAST_VALUE'")
         self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_VALUE'")
         self.basic_query()
 
-        # cache_last 3 
+        # cache_last 3
         self.prepare_datas("'BOTH'")
         self.prepare_tag_datas("'BOTH'")
         self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'BOTH'")

tests/system-test/7-tmq/stbTagFilter.py

@@ -5,103 +5,250 @@ import time
 import socket
 import os
 import threading
+from enum import Enum
 
 from util.log import *
 from util.sql import *
 from util.cases import *
 from util.dnodes import *
-from util.common import *
 sys.path.append("./7-tmq")
 from tmqCommon import *
 
 class TDTestCase:
+    def __init__(self):
+        self.snapshot   = 0
+        self.vgroups    = 4
+        self.ctbNum     = 1
+        self.rowsPerTbl = 10000
+        
     def init(self, conn, logSql):
         tdLog.debug(f"start to excute {__file__}")
-        tdSql.init(conn.cursor())
-        #tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+        tdSql.init(conn.cursor(), False)
 
-    def tmqCase1(self):
-        tdLog.printNoPrefix("======== test case 1: ")
-        paraDict = {'dbName':     'db2',
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
                     'dropFlag':   1,
                     'event':      '',
-                    'vgroups':    1,
+                    'vgroups':    4,
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
                     'ctbPrefix':  'ctb',
                     'ctbStartIdx': 0,
-                    'ctbNum':     10,
-                    'rowsPerTbl': 1000,
-                    'batchNum':   10,
+                    'ctbNum':     1,
+                    'rowsPerTbl': 100000,
+                    'batchNum':   1200,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    'pollDelay':  10,
+                    'pollDelay':  3,
                     'showMsg':    1,
-                    'showRow':    1}
+                    'showRow':    1,
+                    'snapshot':   0}
 
-        topicNameList = ['topic1']
-        expectRowsList = []
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
         tmqCom.initConsumerTable()
-        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=1,replica=1)
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
         tdLog.info("create stb")
-        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
         tdLog.info("create ctb")
-        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx'])
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
         tdLog.info("insert data")
-        tmqCom.asyncInsertData(paraDict)
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx",
+        #                                       ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+        #                                       startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        # tdLog.info("restart taosd to ensure that the data falls into the disk")        
+        # tdSql.query("flush database %s"%(paraDict['dbName']))
+        return
+   
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     1,
+                    'rowsPerTbl': 100000,
+                    'batchNum':   3000,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  5,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+        paraDict['snapshot'] = self.snapshot
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        # update to half tables
+        # paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2)
+        # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx",
+        #                                       ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+        #                                       startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+        #                                        ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+        #                                        startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])     
+        
+        tdLog.info("create topics from stb1")
+        topicFromStb1 = 'topic_stb1'                
+        queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
+        # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicFromStb1, queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)        
+        
+        # paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        consumerId     = 0
+        expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2)
+        topicList      = topicFromStb1
+        ifcheckdata    = 1
+        ifManualCommit = 1
+        keyList        = 'group.id:cgrp1,\
+                        enable.auto.commit:true,\
+                        auto.commit.interval.ms:1000,\
+                        auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
 
-        tdLog.info("create topics from stb with filter")
-        # queryString = "select ts, sin(c1), pow(c2,3) from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName'])
-        queryString = "select * from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName'])
-        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("insert process end, and start to check consume result")
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+
+        tdLog.info("run select sql from db")
+        tdSql.query(queryString)
+        expectrowcnt = tdSql.getRows()
+                
+        tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+        if totalConsumeRows != expectrowcnt:
+            tdLog.exit("tmq consume rows error!")
+            
+        tmqCom.checkFileContent(consumerId, queryString) 
+
+        tdSql.query("drop topic %s"%topicFromStb1)
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def tmqCase2(self):
+        tdLog.printNoPrefix("======== test case 2: ")  
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     1,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   5000,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  5,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+        
+        paraDict['snapshot'] = self.snapshot
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        tdLog.info("restart taosd to ensure that the data falls into the disk")        
+        tdSql.query("flush database %s"%(paraDict['dbName']))
+        
+        # update to half tables
+        paraDict['startTs'] = paraDict['startTs'] + int(self.rowsPerTbl / 2)
+        paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2)
+        tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                              ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                              startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+        #                                        ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+        #                                        startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])     
+
+        tmqCom.initConsumerTable()
+        tdLog.info("create topics from stb1")
+        topicFromStb1 = 'topic_stb1'                
+        queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicFromStb1, queryString)
         tdLog.info("create topic sql: %s"%sqlString)
-        tdSql.execute(sqlString) 
-          
-        # start tmq consume processor
-        tdLog.info("insert consume info to consume processor")
-        consumerId   = 0
-        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
-        topicList    = topicNameList[0]
-        ifcheckdata  = 0
+        tdSql.execute(sqlString)
+        
+        # paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        consumerId     = 1
+        expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2)
+        topicList      = topicFromStb1
+        ifcheckdata    = 1
         ifManualCommit = 1
-        keyList      = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:2000, auto.offset.reset:earliest'
+        keyList        = 'group.id:cgrp1,\
+                        enable.auto.commit:true,\
+                        auto.commit.interval.ms:1000,\
+                        auto.offset.reset:earliest'
         tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
 
         tdLog.info("start consume processor")
-        tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
-
-        # tmqCom.getStartCommitNotifyFromTmqsim()
-        tmqCom.getStartConsumeNotifyFromTmqsim()
-        tdLog.info("create some new ctb")
-        paraDict['ctbStartIdx'] = paraDict['ctbStartIdx'] + paraDict['ctbNum']
-        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx'])
-        tdLog.info("insert data into new ctb")
-        pThread = tmqCom.asyncInsertData(paraDict)
-
-        pThread.join()
-        tdLog.info("wait insert end") 
-        tdSql.query(queryString)        
-        expectRowsList.append(tdSql.getRows())
-
-        tdLog.info("wait the consume result") 
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        
+        tdLog.info("insert process end, and start to check consume result")
         expectRows = 1
         resultList = tmqCom.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+
+        tdSql.query(queryString)
+        totalRowsInserted = tdSql.getRows()
         
-        if expectRowsList[0] != resultList[0]:
-            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
-            tdLog.exit("0 tmq consume rows error!")
+        tdLog.info("act consume rows: %d, act insert rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsInserted, expectrowcnt))
         
-        time.sleep(10)        
-        for i in range(len(topicNameList)):
-            tdSql.query("drop topic %s"%topicNameList[i])
+        if totalConsumeRows != expectrowcnt:
+            tdLog.exit("tmq consume rows error!")
+            
+        # tmqCom.checkFileContent(consumerId, queryString)   
 
-        tdLog.printNoPrefix("======== test case 1 end ...... ")
+        tdSql.query("drop topic %s"%topicFromStb1)
+
+        tdLog.printNoPrefix("======== test case 2 end ...... ")
 
     def run(self):
         tdSql.prepare()
+        self.prepareTestEnv()
+        tdLog.printNoPrefix("=============================================")
+        tdLog.printNoPrefix("======== snapshot is 0: only consume from wal")
         self.tmqCase1()
+        # self.tmqCase2()
+        
+        # self.prepareTestEnv()
+        # tdLog.printNoPrefix("====================================================================")
+        # tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal")
+        # self.snapshot = 1
+        # self.tmqCase1()
+        # self.tmqCase2()
+        
 
     def stop(self):
         tdSql.close()

tests/system-test/99-TDcase/TD-17255.py

@@ -320,7 +320,7 @@ class TDTestCase:
         tdSql.prepare()
 
         self.tmqCase1()
-        # self.tmqCase2() 
+        self.tmqCase2() 
         self.tmqCase3()
 
     def stop(self):

tests/system-test/fulltest.sh

@@ -10,7 +10,7 @@ python3 ./test.py -f 0-others/taosdMonitor.py
 python3 ./test.py -f 0-others/udfTest.py
 python3 ./test.py -f 0-others/udf_create.py
 python3 ./test.py -f 0-others/udf_restart_taosd.py
-python3 ./test.py -f 0-others/cachelast.py
+python3 ./test.py -f 0-others/cachemodel.py
 python3 ./test.py -f 0-others/udf_cfg1.py
 python3 ./test.py -f 0-others/udf_cfg2.py
 
@@ -32,7 +32,10 @@ python3 ./test.py -f 1-insert/block_wise.py
 python3 ./test.py -f 1-insert/create_retentions.py
 python3 ./test.py -f 1-insert/table_param_ttl.py
 
+python3 ./test.py -f 1-insert/update_data.py
+
 python3 ./test.py -f 2-query/db.py
+
 python3 ./test.py -f 2-query/between.py
 python3 ./test.py -f 2-query/distinct.py
 python3 ./test.py -f 2-query/varchar.py

tests/test/c/tmqDemo.c

@@ -596,7 +596,8 @@ void printParaIntoFile() {
   g_fp = pFile;
 
   time_t    tTime = taosGetTimestampSec();
-  struct tm tm = *taosLocalTime(&tTime, NULL);
+  struct tm tm;
+  taosLocalTime(&tTime, &tm);
 
   taosFprintfFile(pFile, "###################################################################\n");
   taosFprintfFile(pFile, "# configDir:                %s\n", configDir);

tests/test/c/tmqSim.c

@@ -171,7 +171,8 @@ static void printHelp() {
 
 char* getCurrentTimeString(char* timeString) {
   time_t    tTime = taosGetTimestampSec();
-  struct tm tm = *taosLocalTime(&tTime, NULL);
+  struct tm tm;
+  taosLocalTime(&tTime, &tm);
   sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour,
           tm.tm_min, tm.tm_sec);
 
@@ -420,18 +421,6 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) {
     ms = val % 1000;
   }
 
-  /*
-    comment out as it make testcases like select_with_tags.sim fail.
-    but in windows, this may cause the call to localtime crash if tt < 0,
-    need to find a better solution.
-    if (tt < 0) {
-      tt = 0;
-    }
-  */
-
-#ifdef WINDOWS
-  if (tt < 0) tt = 0;
-#endif
   if (tt <= 0 && ms < 0) {
     tt--;
     if (precision == TSDB_TIME_PRECISION_NANO) {
@@ -443,8 +432,9 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) {
     }
   }
 
-  struct tm* ptm = taosLocalTime(&tt, NULL);
-  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
+  struct tm ptm;
+  taosLocalTime(&tt, &ptm);
+  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
 
   if (precision == TSDB_TIME_PRECISION_NANO) {
     sprintf(buf + pos, ".%09d", ms);

tests/tsim/src/simExe.c

@@ -635,7 +635,7 @@ bool simCreateTaosdConnect(SScript *script, char *rest) {
 bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
   char       timeStr[30] = {0};
   time_t     tt;
-  struct tm *tp;
+  struct tm  tp;
   SCmdLine  *line = &script->lines[script->linePos];
   int32_t    ret = -1;
 
@@ -768,20 +768,9 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
               } else {
                 tt = (*(int64_t *)row[i]) / 1000000000;
               }
-              /* comment out as it make testcases like select_with_tags.sim fail.
-                but in windows, this may cause the call to localtime crash if tt < 0,
-                need to find a better solution.
-              if (tt < 0) {
-                tt = 0;
-              }
-              */
-
-#ifdef WINDOWS
-              if (tt < 0) tt = 0;
-#endif
 
-              tp = taosLocalTime(&tt, NULL);
-              strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", tp);
+              taosLocalTime(&tt, &tp);
+              strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp);
               if (precision == TSDB_TIME_PRECISION_MILLI) {
                 sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000));
               } else if (precision == TSDB_TIME_PRECISION_MICRO) {

tests/tsim/src/simParse.c

@@ -175,7 +175,7 @@ SScript *simBuildScriptObj(char *fileName) {
 SScript *simParseScript(char *fileName) {
   TdFilePtr pFile;
   int32_t   tokenLen, lineNum = 0;
-  char     *buffer = NULL, name[128], *token, *rest;
+  char      buffer[10*1024], name[128], *token, *rest;
   SCommand *pCmd;
   SScript  *script;
 
@@ -195,7 +195,7 @@ SScript *simParseScript(char *fileName) {
   simResetParser();
 
   while (!taosEOFFile(pFile)) {
-    if (taosGetLineFile(pFile, &buffer) == -1) continue;
+    if (taosGetsFile(pFile, sizeof(buffer) - 1, buffer) == -1) continue;
 
     lineNum++;
     int32_t cmdlen = (int32_t)strlen(buffer);
@@ -240,7 +240,6 @@ SScript *simParseScript(char *fileName) {
       return NULL;
     }
   }
-  if(buffer != NULL) taosMemoryFree(buffer);
   taosCloseFile(&pFile);
 
   script = simBuildScriptObj(fileName);

tools/shell/src/shellEngine.c

@@ -231,18 +231,6 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
     ms = val % 1000;
   }
 
-  /*
-    comment out as it make testcases like select_with_tags.sim fail.
-    but in windows, this may cause the call to localtime crash if tt < 0,
-    need to find a better solution.
-    if (tt < 0) {
-      tt = 0;
-    }
-  */
-
-#ifdef WINDOWS
-  if (tt < 0) tt = 0;
-#endif
   if (tt <= 0 && ms < 0) {
     tt--;
     if (precision == TSDB_TIME_PRECISION_NANO) {
@@ -254,8 +242,9 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
     }
   }
 
-  struct tm *ptm = taosLocalTime(&tt, NULL);
-  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
+  struct tm ptm = {0};
+  taosLocalTime(&tt, &ptm);
+  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
 
   if (precision == TSDB_TIME_PRECISION_NANO) {
     sprintf(buf + pos, ".%09d", ms);