Merge pull request #16663 from taosdata/3.0

release: build 3.0.1.0

source/dnode/vnode/src/inc/sma.h

@@ -148,6 +148,7 @@ struct SRSmaInfoItem {
 };
 
 struct SRSmaInfo {
+  SSma     *pSma;
   STSchema *pTSchema;
   int64_t   suid;
   int64_t   lastRecv;  // ms

source/dnode/vnode/src/inc/tsdb.h

@@ -65,12 +65,14 @@ typedef struct SBlockInfo    SBlockInfo;
 typedef struct SSmaInfo      SSmaInfo;
 typedef struct SBlockCol     SBlockCol;
 typedef struct SVersionRange SVersionRange;
+typedef struct SLDataIter    SLDataIter;
 
 #define TSDB_FILE_DLMT         ((uint32_t)0xF00AFA0F)
 #define TSDB_MAX_SUBBLOCKS     8
-#define TSDB_MAX_LAST_FILE     16
-#define TSDB_DEFAULT_LAST_FILE 8
+#define TSDB_MAX_SST_FILE      16
+#define TSDB_DEFAULT_SST_FILE  8
 #define TSDB_FHDR_SIZE         512
+#define TSDB_DEFAULT_PAGE_SIZE 4096
 
 #define HAS_NONE  ((int8_t)0x1)
 #define HAS_NULL  ((int8_t)0x2)
@@ -82,6 +84,14 @@ typedef struct SVersionRange SVersionRange;
 #define TSDBKEY_MIN ((TSDBKEY){.ts = TSKEY_MIN, .version = VERSION_MIN})
 #define TSDBKEY_MAX ((TSDBKEY){.ts = TSKEY_MAX, .version = VERSION_MAX})
 
+#define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM))
+#define LOGIC_TO_FILE_OFFSET(LOFFSET, PAGE) \
+  ((LOFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (LOFFSET) % PAGE_CONTENT_SIZE(PAGE))
+#define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE)*PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE))
+#define PAGE_OFFSET(PGNO, PAGE)            (((PGNO)-1) * (PAGE))
+#define OFFSET_PGNO(OFFSET, PAGE)          ((OFFSET) / (PAGE) + 1)
+#define LOGIC_TO_FILE_SIZE(LSIZE, PAGE)    OFFSET_PGNO(LOGIC_TO_FILE_OFFSET(LSIZE, PAGE), PAGE) * (PAGE)
+
 // tsdbUtil.c ==============================================================================================
 // TSDBROW
 #define TSDBROW_TS(ROW)                       (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow])
@@ -195,7 +205,6 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol
                         uint8_t **ppBuf);
 int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData,
                           uint8_t **ppBuf);
-int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck);
 // tsdbMemTable ==============================================================================================
 // SMemTable
 int32_t  tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
@@ -563,7 +572,7 @@ struct SDFileSet {
   SDataFile *pDataF;
   SSmaFile  *pSmaF;
   uint8_t    nSstF;
-  SSstFile  *aSstF[TSDB_MAX_LAST_FILE];
+  SSstFile  *aSstF[TSDB_MAX_SST_FILE];
 };
 
 struct SRowIter {
@@ -578,46 +587,54 @@ struct SRowMerger {
   SArray   *pArray;  // SArray<SColVal>
 };
 
-struct SDelFWriter {
-  STsdb    *pTsdb;
-  SDelFile  fDel;
-  TdFilePtr pWriteH;
+typedef struct {
+  char     *path;
+  int32_t   szPage;
+  int32_t   flag;
+  TdFilePtr pFD;
+  int64_t   pgno;
+  uint8_t  *pBuf;
+  int64_t   szFile;
+} STsdbFD;
 
+struct SDelFWriter {
+  STsdb   *pTsdb;
+  SDelFile fDel;
+  STsdbFD *pWriteH;
   uint8_t *aBuf[1];
 };
 
+struct STsdbReadSnap {
+  SMemTable *pMem;
+  SMemTable *pIMem;
+  STsdbFS    fs;
+};
+
 struct SDataFWriter {
   STsdb    *pTsdb;
   SDFileSet wSet;
 
-  TdFilePtr pHeadFD;
-  TdFilePtr pDataFD;
-  TdFilePtr pSmaFD;
-  TdFilePtr pLastFD;
+  STsdbFD *pHeadFD;
+  STsdbFD *pDataFD;
+  STsdbFD *pSmaFD;
+  STsdbFD *pSstFD;
 
   SHeadFile fHead;
   SDataFile fData;
   SSmaFile  fSma;
-  SSstFile  fSst[TSDB_MAX_LAST_FILE];
+  SSstFile  fSst[TSDB_MAX_SST_FILE];
 
   uint8_t *aBuf[4];
 };
 
-struct STsdbReadSnap {
-  SMemTable *pMem;
-  SMemTable *pIMem;
-  STsdbFS    fs;
-};
-
 struct SDataFReader {
   STsdb     *pTsdb;
   SDFileSet *pSet;
-  TdFilePtr  pHeadFD;
-  TdFilePtr  pDataFD;
-  TdFilePtr  pSmaFD;
-  TdFilePtr  aLastFD[TSDB_MAX_LAST_FILE];
-
-  uint8_t *aBuf[3];
+  STsdbFD   *pHeadFD;
+  STsdbFD   *pDataFD;
+  STsdbFD   *pSmaFD;
+  STsdbFD   *aSstFD[TSDB_MAX_SST_FILE];
+  uint8_t   *aBuf[3];
 };
 
 typedef struct {
@@ -627,15 +644,16 @@ typedef struct {
 } SRowInfo;
 
 typedef struct SMergeTree {
-  int8_t             backward;
-  SRBTree            rbt;
-  SArray            *pIterList;
-  struct SLDataIter *pIter;
+  int8_t      backward;
+  SRBTree     rbt;
+  SArray     *pIterList;
+  SLDataIter *pIter;
 } SMergeTree;
 
-int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange);
-void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter);
-bool tMergeTreeNext(SMergeTree *pMTree);
+int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t uid,
+                       STimeWindow *pTimeWindow, SVersionRange *pVerRange);
+void    tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
+bool    tMergeTreeNext(SMergeTree *pMTree);
 TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
 void    tMergeTreeClose(SMergeTree *pMTree);
 

source/dnode/vnode/src/sma/smaCommit.c

@@ -188,13 +188,15 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) {
 
   for (int32_t i = 0; i < taosArrayGetSize(pFS->aQTaskInf);) {
     SQTaskFile *pTaskF = taosArrayGet(pFS->aQTaskInf, i);
-    if (atomic_sub_fetch_32(&pTaskF->nRef, 1) <= 0) {
+    int32_t     oldVal = atomic_fetch_sub_32(&pTaskF->nRef, 1);
+    if ((oldVal <= 1) && (pTaskF->version < committed)) {
       tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName);
       if (taosRemoveFile(qTaskInfoFullName) < 0) {
-        smaWarn("vgId:%d, cleanup qinf, failed to remove %s since %s", TD_VID(pVnode), qTaskInfoFullName,
-                tstrerror(TAOS_SYSTEM_ERROR(errno)));
+        smaWarn("vgId:%d, cleanup qinf, committed %" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed,
+                qTaskInfoFullName, tstrerror(TAOS_SYSTEM_ERROR(errno)));
       } else {
-        smaDebug("vgId:%d, cleanup qinf, success to remove %s", TD_VID(pVnode), qTaskInfoFullName);
+        smaDebug("vgId:%d, cleanup qinf, committed %" PRIi64 ", success to remove %s", TD_VID(pVnode), committed,
+                 qTaskInfoFullName);
       }
       taosArrayRemove(pFS->aQTaskInf, i);
       continue;
@@ -364,7 +366,6 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
   }
 
   SRSmaStat     *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
-  SRSmaInfoItem *pItem = NULL;
 
   // step 1: merge qTaskInfo and iQTaskInfo
   // lock

source/dnode/vnode/src/sma/smaEnv.c

@@ -62,7 +62,7 @@ int32_t smaInit() {
     }
 
     int32_t type = (8 == POINTER_BYTES) ? TSDB_DATA_TYPE_UBIGINT : TSDB_DATA_TYPE_UINT;
-    smaMgmt.refHash = taosHashInit(1, taosGetDefaultHashFunction(type), true, HASH_ENTRY_LOCK);
+    smaMgmt.refHash = taosHashInit(64, taosGetDefaultHashFunction(type), true, HASH_ENTRY_LOCK);
     if (!smaMgmt.refHash) {
       taosCloseRef(smaMgmt.rsetId);
       atomic_store_8(&smaMgmt.inited, 0);
@@ -107,6 +107,7 @@ void smaCleanUp() {
   if (old == 1) {
     taosCloseRef(smaMgmt.rsetId);
     taosHashCleanup(smaMgmt.refHash);
+    smaMgmt.refHash = NULL;
     taosTmrCleanUp(smaMgmt.tmrHandle);
     smaInfo("sma mgmt env is cleaned up, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle);
     atomic_store_8(&smaMgmt.inited, 0);
@@ -220,7 +221,7 @@ static void tRSmaInfoHashFreeNode(void *data) {
     if ((pItem = RSMA_INFO_ITEM((SRSmaInfo *)pRSmaInfo, 1)) && pItem->level) {
       taosHashRemove(smaMgmt.refHash, &pItem, POINTER_BYTES);
     }
-    tdFreeRSmaInfo(NULL, pRSmaInfo, true);
+    tdFreeRSmaInfo(pRSmaInfo->pSma, pRSmaInfo, true);
   }
 }
 

source/dnode/vnode/src/sma/smaRollup.c

@@ -121,27 +121,27 @@ static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t l
  */
 void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) {
   if (pInfo) {
-    int32_t vid = pSma ? SMA_VID(pSma) : -1;
     for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
 
       if (isDeepFree && pItem->tmrId) {
-        smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", vid, pItem->tmrId, pInfo->suid, i + 1);
+        smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pItem->tmrId,
+                 pInfo->suid, i + 1);
         taosTmrStopA(&pItem->tmrId);
       }
 
       if (isDeepFree && pInfo->taskInfo[i]) {
-        tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], vid, i + 1);
+        tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1);
       } else {
-        smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", vid,
+        smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma),
                  pInfo->suid, i + 1);
       }
 
       if (pInfo->iTaskInfo[i]) {
-        tdRSmaQTaskInfoFree(&pInfo->iTaskInfo[i], vid, i + 1);
+        tdRSmaQTaskInfoFree(&pInfo->iTaskInfo[i], SMA_VID(pSma), i + 1);
       } else {
-        smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty iTaskInfo", vid,
-                 pInfo->suid, i + 1);
+        smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty iTaskInfo",
+                 SMA_VID(pSma), pInfo->suid, i + 1);
       }
     }
     if (isDeepFree) {
@@ -377,7 +377,10 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
     terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
     goto _err;
   }
+  pRSmaInfo->pSma = pSma;
   pRSmaInfo->pTSchema = pTSchema;
+  pRSmaInfo->suid = suid;
+  T_REF_INIT_VAL(pRSmaInfo, 1);
   if (!(pRSmaInfo->queue = taosOpenQueue())) {
     goto _err;
   }
@@ -391,8 +394,6 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
   if (!(pRSmaInfo->iQall = taosAllocateQall())) {
     goto _err;
   }
-  pRSmaInfo->suid = suid;
-  T_REF_INIT_VAL(pRSmaInfo, 1);
 
   if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) {
     goto _err;
@@ -1586,8 +1587,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
 
   if (!(pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaRef->refId))) {
     smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%" PRIi64 " refId:%d)",
-             smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove
-    taosHashRemove(smaMgmt.refHash, &param, POINTER_BYTES); 
+             smaMgmt.rsetId, pRSmaRef->refId);  // pRSmaRef freed in taosHashRemove
+    taosHashRemove(smaMgmt.refHash, &param, POINTER_BYTES);
     return;
   }
 
@@ -1636,7 +1637,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
   switch (fetchTriggerStat) {
     case TASK_TRIGGER_STAT_ACTIVE: {
       smaDebug("vgId:%d, rsma fetch task planned for level:%" PRIi8 " suid:%" PRIi64 " since stat is active",
-              SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+               SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
       // async process
       pItem->fetchLevel = pItem->level;
 #if 0

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -71,7 +71,7 @@ typedef struct {
     SDataIter *pIter;
     SRBTree    rbt;
     SDataIter  dataIter;
-    SDataIter  aDataIter[TSDB_MAX_LAST_FILE];
+    SDataIter  aDataIter[TSDB_MAX_SST_FILE];
     int8_t     toLastOnly;
   };
   struct {
@@ -92,9 +92,6 @@ typedef struct {
   SArray      *aDelData;  // SArray<SDelData>
 } SCommitter;
 
-extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *aSstBlk,
-                                  SBlockData *pBlockData);  // todo
-
 static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter);
 static int32_t tsdbCommitData(SCommitter *pCommitter);
 static int32_t tsdbCommitDel(SCommitter *pCommitter);
@@ -445,7 +442,7 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
 
         pIter->iSstBlk = 0;
         SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, 0);
-        code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, iSst, pSstBlk, &pIter->bData);
+        code = tsdbReadSstBlock(pCommitter->dReader.pReader, iSst, pSstBlk, &pIter->bData);
         if (code) goto _err;
 
         pIter->iRow = 0;
@@ -760,7 +757,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
   pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
   pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
   pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
-  pCommitter->maxLast = TSDB_DEFAULT_LAST_FILE;  // TODO: make it as a config
+  pCommitter->maxLast = TSDB_DEFAULT_SST_FILE;  // TODO: make it as a config
   pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem);
   if (pCommitter->aTbDataP == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
@@ -790,7 +787,7 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
   if (code) goto _exit;
 
   // merger
-  for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) {
+  for (int32_t iSst = 0; iSst < TSDB_MAX_SST_FILE; iSst++) {
     SDataIter *pIter = &pCommitter->aDataIter[iSst];
     pIter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
     if (pIter->aSstBlk == NULL) {
@@ -832,7 +829,7 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) {
   tBlockDataDestroy(&pCommitter->dReader.bData, 1);
 
   // merger
-  for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) {
+  for (int32_t iSst = 0; iSst < TSDB_MAX_SST_FILE; iSst++) {
     SDataIter *pIter = &pCommitter->aDataIter[iSst];
     taosArrayDestroy(pIter->aSstBlk);
     tBlockDataDestroy(&pIter->bData, 1);
@@ -1059,7 +1056,7 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) {
         if (pIter->iSstBlk < taosArrayGetSize(pIter->aSstBlk)) {
           SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk);
 
-          code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, pIter->iSst, pSstBlk, &pIter->bData);
+          code = tsdbReadSstBlock(pCommitter->dReader.pReader, pIter->iSst, pSstBlk, &pIter->bData);
           if (code) goto _exit;
 
           pIter->iRow = 0;

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -21,6 +21,9 @@ static int32_t tsdbEncodeFS(uint8_t *p, STsdbFS *pFS) {
   int8_t   hasDel = pFS->pDelFile ? 1 : 0;
   uint32_t nSet = taosArrayGetSize(pFS->aDFileSet);
 
+  // version
+  n += tPutI8(p ? p + n : p, 0);
+
   // SDelFile
   n += tPutI8(p ? p + n : p, hasDel);
   if (hasDel) {
@@ -292,7 +295,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
       code = TAOS_SYSTEM_ERROR(errno);
       goto _err;
     }
-    if (size != pSet->pHeadF->size) {
+    if (size != LOGIC_TO_FILE_SIZE(pSet->pHeadF->size, TSDB_DEFAULT_PAGE_SIZE)) {
       code = TSDB_CODE_FILE_CORRUPTED;
       goto _err;
     }
@@ -303,10 +306,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
       code = TAOS_SYSTEM_ERROR(errno);
       goto _err;
     }
-    if (size < pSet->pDataF->size) {
+    if (size < LOGIC_TO_FILE_SIZE(pSet->pDataF->size, TSDB_DEFAULT_PAGE_SIZE)) {
       code = TSDB_CODE_FILE_CORRUPTED;
       goto _err;
-    } else if (size > pSet->pDataF->size) {
+    } else if (size > LOGIC_TO_FILE_SIZE(pSet->pDataF->size, TSDB_DEFAULT_PAGE_SIZE)) {
       code = tsdbDFileRollback(pTsdb, pSet, TSDB_DATA_FILE);
       if (code) goto _err;
     }
@@ -317,10 +320,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
       code = TAOS_SYSTEM_ERROR(errno);
       goto _err;
     }
-    if (size < pSet->pSmaF->size) {
+    if (size < LOGIC_TO_FILE_SIZE(pSet->pSmaF->size, TSDB_DEFAULT_PAGE_SIZE)) {
       code = TSDB_CODE_FILE_CORRUPTED;
       goto _err;
-    } else if (size > pSet->pSmaF->size) {
+    } else if (size > LOGIC_TO_FILE_SIZE(pSet->pSmaF->size, TSDB_DEFAULT_PAGE_SIZE)) {
       code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE);
       if (code) goto _err;
     }
@@ -332,7 +335,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
         code = TAOS_SYSTEM_ERROR(errno);
         goto _err;
       }
-      if (size != pSet->aSstF[iSst]->size) {
+      if (size != LOGIC_TO_FILE_SIZE(pSet->aSstF[iSst]->size, TSDB_DEFAULT_PAGE_SIZE)) {
         code = TSDB_CODE_FILE_CORRUPTED;
         goto _err;
       }
@@ -364,10 +367,12 @@ static int32_t tsdbRecoverFS(STsdb *pTsdb, uint8_t *pData, int64_t nData) {
   int32_t  code = 0;
   int8_t   hasDel;
   uint32_t nSet;
-  int32_t  n;
+  int32_t  n = 0;
+
+  // version
+  n += tGetI8(pData + n, NULL);
 
   // SDelFile
-  n = 0;
   n += tGetI8(pData + n, &hasDel);
   if (hasDel) {
     pTsdb->fs.pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile));

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -148,7 +148,7 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) {
   }
 
   // ftruncate
-  if (taosFtruncateFile(pFD, size) < 0) {
+  if (taosFtruncateFile(pFD, LOGIC_TO_FILE_SIZE(size, TSDB_DEFAULT_PAGE_SIZE)) < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -16,7 +16,7 @@
 #include "tsdb.h"
 
 // SLDataIter =================================================
-typedef struct SLDataIter {
+struct SLDataIter {
   SRBTreeNode   node;
   SSstBlk      *pSstBlk;
   SDataFReader *pReader;
@@ -31,13 +31,11 @@ typedef struct SLDataIter {
   uint64_t      uid;
   STimeWindow   timeWindow;
   SVersionRange verRange;
-} SLDataIter;
+};
 
-static SBlockData* getCurrentBlock(SLDataIter* pIter) {
-  return &pIter->bData[pIter->loadIndex];
-}
+static SBlockData *getCurrentBlock(SLDataIter *pIter) { return &pIter->bData[pIter->loadIndex]; }
 
-static SBlockData* getNextBlock(SLDataIter* pIter) {
+static SBlockData *getNextBlock(SLDataIter *pIter) {
   pIter->loadIndex ^= 1;
   return getCurrentBlock(pIter);
 }
@@ -116,8 +114,6 @@ void tLDataIterClose(SLDataIter *pIter) {
   taosMemoryFree(pIter);
 }
 
-extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData);
-
 void tLDataIterNextBlock(SLDataIter *pIter) {
   int32_t step = pIter->backward ? -1 : 1;
   pIter->iSstBlk += step;
@@ -150,9 +146,9 @@ void tLDataIterNextBlock(SLDataIter *pIter) {
 static void findNextValidRow(SLDataIter *pIter) {
   int32_t step = pIter->backward ? -1 : 1;
 
-  bool    hasVal = false;
-  int32_t i = pIter->iRow;
-  SBlockData* pBlockData = getCurrentBlock(pIter);
+  bool        hasVal = false;
+  int32_t     i = pIter->iRow;
+  SBlockData *pBlockData = getCurrentBlock(pIter);
 
   for (; i < pBlockData->nRow && i >= 0; i += step) {
     if (pBlockData->aUid != NULL) {
@@ -220,12 +216,12 @@ bool tLDataIterNextRow(SLDataIter *pIter) {
     return false;
   }
 
-  int32_t iBlockL = pIter->iSstBlk;
-  SBlockData* pBlockData = getCurrentBlock(pIter);
+  int32_t     iBlockL = pIter->iSstBlk;
+  SBlockData *pBlockData = getCurrentBlock(pIter);
 
   if (pBlockData->nRow == 0 && pIter->pSstBlk != NULL) {  // current block not loaded yet
     pBlockData = getNextBlock(pIter);
-    code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData);
+    code = tsdbReadSstBlock(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData);
     if (code != TSDB_CODE_SUCCESS) {
       goto _exit;
     }
@@ -249,7 +245,7 @@ bool tLDataIterNextRow(SLDataIter *pIter) {
 
     if (iBlockL != pIter->iSstBlk) {
       pBlockData = getNextBlock(pIter);
-      code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData);
+      code = tsdbReadSstBlock(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData);
       if (code) {
         goto _exit;
       }
@@ -306,7 +302,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
   tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn);
   int32_t code = TSDB_CODE_OUT_OF_MEMORY;
 
-  struct SLDataIter *pIterList[TSDB_DEFAULT_LAST_FILE] = {0};
+  struct SLDataIter *pIterList[TSDB_DEFAULT_SST_FILE] = {0};
   for (int32_t i = 0; i < pFReader->pSet->nSstF; ++i) {  // open all last file
     code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange);
     if (code != TSDB_CODE_SUCCESS) {

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -1548,7 +1548,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut,
       if (code) goto _exit;
 
       blockCol.offset = aBufN[0];
-      aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM);
+      aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue;
     }
 
     code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol));
@@ -1556,15 +1556,8 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut,
     hdr.szBlkCol += tPutBlockCol(aBuf[1] + hdr.szBlkCol, &blockCol);
   }
 
-  aBufN[1] = 0;
-  if (hdr.szBlkCol > 0) {
-    aBufN[1] = hdr.szBlkCol + sizeof(TSCKSUM);
-
-    code = tRealloc(&aBuf[1], aBufN[1]);
-    if (code) goto _exit;
-
-    taosCalcChecksumAppend(0, aBuf[1], aBufN[1]);
-  }
+  // SBlockCol
+  aBufN[1] = hdr.szBlkCol;
 
   // uid + version + tskey
   aBufN[2] = 0;
@@ -1585,16 +1578,11 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut,
   if (code) goto _exit;
   aBufN[2] += hdr.szKey;
 
-  aBufN[2] += sizeof(TSCKSUM);
-  code = tRealloc(&aBuf[2], aBufN[2]);
-  if (code) goto _exit;
-
   // hdr
   aBufN[3] = tPutDiskDataHdr(NULL, &hdr);
   code = tRealloc(&aBuf[3], aBufN[3]);
   if (code) goto _exit;
   tPutDiskDataHdr(aBuf[3], &hdr);
-  taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], aBufN[3]), aBuf[2], aBufN[2]);
 
   // aggragate
   if (ppOut) {
@@ -1626,10 +1614,6 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
 
   // SDiskDataHdr
   n += tGetDiskDataHdr(pIn + n, &hdr);
-  if (!taosCheckChecksumWhole(pIn, n + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM))) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _exit;
-  }
   ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
 
   pBlockData->suid = hdr.suid;
@@ -1657,7 +1641,7 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
   code = tsdbDecmprData(pIn + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY,
                         sizeof(TSKEY) * hdr.nRow, &aBuf[0]);
   if (code) goto _exit;
-  n = n + hdr.szKey + sizeof(TSCKSUM);
+  n += hdr.szKey;
 
   // loop to decode each column data
   if (hdr.szBlkCol == 0) goto _exit;
@@ -1679,8 +1663,8 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
         if (code) goto _exit;
       }
     } else {
-      code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &blockCol, hdr.cmprAlg,
-                               hdr.nRow, pColData, &aBuf[0]);
+      code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData,
+                               &aBuf[0]);
       if (code) goto _exit;
     }
   }
@@ -2062,12 +2046,6 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol
   }
   size += pBlockCol->szValue;
 
-  // checksum
-  size += sizeof(TSCKSUM);
-  code = tRealloc(ppOut, nOut + size);
-  if (code) goto _exit;
-  taosCalcChecksumAppend(0, *ppOut + nOut, size);
-
 _exit:
   return code;
 }
@@ -2076,12 +2054,6 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in
                           uint8_t **ppBuf) {
   int32_t code = 0;
 
-  int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM);
-  if (!taosCheckChecksumWhole(pIn, size)) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _exit;
-  }
-
   ASSERT(pColData->cid == pBlockCol->cid);
   ASSERT(pColData->type == pBlockCol->type);
   pColData->smaOn = pBlockCol->smaOn;
@@ -2153,37 +2125,3 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in
 _exit:
   return code;
 }
-
-int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) {
-  int32_t code = 0;
-
-  // alloc
-  code = tRealloc(ppOut, size);
-  if (code) goto _exit;
-
-  // seek
-  int64_t n = taosLSeekFile(pFD, offset, SEEK_SET);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _exit;
-  }
-
-  // read
-  n = taosReadFile(pFD, *ppOut, size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _exit;
-  } else if (n < size) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _exit;
-  }
-
-  // check
-  if (toCheck && !taosCheckChecksumWhole(*ppOut, size)) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _exit;
-  }
-
-_exit:
-  return code;
-}

source/libs/executor/inc/executorimpl.h

@@ -591,6 +591,24 @@ typedef struct SMergeAlignedIntervalAggOperatorInfo {
   SNode*       pCondition;
 } SMergeAlignedIntervalAggOperatorInfo;
 
+typedef struct SStreamIntervalOperatorInfo {
+  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
+  SOptrBasicInfo     binfo;              // basic info
+  SAggSupporter      aggSup;             // aggregate supporter
+  SExprSupp          scalarSupp;         // supporter for perform scalar function
+  SGroupResInfo      groupResInfo;       // multiple results build supporter
+  SInterval          interval;           // interval info
+  int32_t            primaryTsIndex;     // primary time stamp slot id from result of downstream operator.
+  STimeWindowAggSupp twAggSup;
+  bool               invertible;
+  bool               ignoreExpiredData;
+  SArray*            pRecycledPages;
+  SArray*            pDelWins;           // SWinRes
+  int32_t            delIndex;
+  SSDataBlock*       pDelRes;
+  bool               isFinal;
+} SStreamIntervalOperatorInfo;
+
 typedef struct SStreamFinalIntervalOperatorInfo {
   // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
   SOptrBasicInfo     binfo;           // basic info
@@ -909,8 +927,7 @@ void    initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLi
 void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset,
                       int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput);
 
-int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList,
-                                     char** pNextStart);
+int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart);
 void    updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs,
                              SOperatorInfo* pOperator);
 
@@ -1004,6 +1021,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
                                                   SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                        SExecTaskInfo* pTaskInfo, int32_t numOfChild);
+SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream,
+                                                SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                 SExecTaskInfo* pTaskInfo);

source/libs/executor/src/executor.c

@@ -97,6 +97,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
   }
 }
 
+static FORCE_INLINE void streamInputBlockDataDestory(void* pBlock) { blockDataDestroy((SSDataBlock*)pBlock); }
+
 void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
   if (!pTaskInfo || !pTaskInfo->pRoot || pTaskInfo->pRoot->numOfDownstream <= 0) {
@@ -107,11 +109,7 @@ void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo) {
   if (pOptrInfo->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
     SStreamScanInfo* pInfo = pOptrInfo->info;
     if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) {
-      for (int32_t i = 0; i < taosArrayGetSize(pInfo->pBlockLists); ++i) {
-        SSDataBlock* p = *(SSDataBlock**)taosArrayGet(pInfo->pBlockLists, i);
-        taosArrayDestroy(p->pDataBlock);
-        taosMemoryFreeClear(p);
-      }
+      taosArrayClearP(pInfo->pBlockLists, streamInputBlockDataDestory);
     } else {
       ASSERT(0);
     }

source/libs/executor/src/executorimpl.c

@@ -1866,13 +1866,11 @@ void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t
   pOperator->resultInfo.totalRows += numOfRows;
 }
 
-int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList,
-                                     char** pNextStart) {
+int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) {
   if (pColList == NULL) {  // data from other sources
     blockDataCleanup(pRes);
     *pNextStart = (char*)blockDecode(pRes, pData);
   } else {  // extract data according to pColList
-    ASSERT(numOfOutput == taosArrayGetSize(pColList));
     char* pStart = pData;
 
     int32_t numOfCols = htonl(*(int32_t*)pStart);
@@ -1970,7 +1968,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
       char*              pStart = pRetrieveRsp->data;
       while (index++ < pRetrieveRsp->numOfBlocks) {
         SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false);
-        code = extractDataBlockFromFetchRsp(pb, pStart, pRetrieveRsp->numOfCols, NULL, &pStart);
+        code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart);
         if (code != 0) {
           taosMemoryFreeClear(pDataInfo->pRsp);
           goto _error;
@@ -2095,7 +2093,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
     SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp;
 
     char*   pStart = pRetrieveRsp->data;
-    int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, pRetrieveRsp->numOfCols, NULL, &pStart);
+    int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart);
 
     if (pRsp->completed == 1) {
       qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64
@@ -3915,7 +3913,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions,
                                           pScalarExprInfo, numOfScalarExpr, pAggNode->mergeDataBlock, pTaskInfo);
     }
-  } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
+  } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) {
     SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
 
     SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
@@ -3940,6 +3938,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode,
                                        pTaskInfo, isStream);
 
+  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { 
+    pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) {
     SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode;
     pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo);

source/libs/executor/src/scanoperator.c

@@ -2649,7 +2649,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
       }
 
       char* pStart = pRsp->data;
-      extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pOperator->exprSupp.numOfExprs, pInfo->scanCols, &pStart);
+      extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->scanCols, &pStart);
       updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator);
 
       // todo log the filter info

tests/script/tsim/stream/basic1.sim

@@ -588,4 +588,38 @@ if $data00 != 5 then
   goto loop3
 endi
 
+#max,min selectivity
+sql create database test3  vgroups 1;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from ts1 interval(10s) ;
+
+sql insert into ts1 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts1 values(1648791222001,2,2,3);
+sleep 50
+
+$loop_count = 0
+loop3:
+sql select * from streamtST3;
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+# row 0
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop3
+endi
+
+# row 1
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop3
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stream/distributeInterval0.sim

@@ -198,7 +198,7 @@ endi
 sql select  _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b)  c4, min(c) c5, avg(d) from st interval(10s);
 
 
-sql create database test1  vgroups 1;
+sql create database test1  vgroups 4;
 sql use test1;
 sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
 sql create table ts1 using st tags(1,1,1);
@@ -232,4 +232,43 @@ if $data11 != 2 then
   goto loop2
 endi
 
+#max,min selectivity
+sql create database test3  vgroups 4;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create table ts2 using st tags(2,2,2);
+sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ;
+
+sql insert into ts1 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts1 values(1648791222001,2,2,3);
+sleep 50
+sql insert into ts2 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts2 values(1648791222001,2,2,3);
+sleep 50
+
+$loop_count = 0
+loop3:
+sql select * from streamtST3;
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+# row 0
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop3
+endi
+
+# row 1
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop3
+endi
+
 system sh/stop_dnodes.sh

tests/script/tsim/stream/partitionbyColumn0.sim

@@ -24,7 +24,7 @@ sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop0:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -48,7 +48,7 @@ sql insert into t1 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop1:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -71,7 +71,7 @@ sql insert into t1 values(1648791213000,2,2,3,1.0);
 $loop_count = 0
 
 loop2:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -97,7 +97,7 @@ sql insert into t1 values(1648791213002,1,2,3,1.0);
 $loop_count = 0
 
 loop3:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -134,7 +134,7 @@ sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (16
 $loop_count = 0
 
 loop4:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -208,7 +208,7 @@ sql insert into t1 values(1648791213001,1,2,3,2.0);
 $loop_count = 0
 
 loop5:
-sleep 100
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -229,7 +229,7 @@ sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
 $loop_count = 0
 
 loop6:
-sleep 100
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -294,7 +294,7 @@ sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop7:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -318,7 +318,7 @@ sql insert into t2 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop8:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -342,7 +342,7 @@ sql insert into t2 values(1648791213000,2,2,3,1.0);
 $loop_count = 0
 
 loop9:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -372,7 +372,7 @@ sql insert into t2 values(1648791213002,1,2,3,1.0);
 $loop_count = 0
 
 loop10:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -414,7 +414,7 @@ sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (16
 $loop_count = 0
 
 loop11:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -492,7 +492,7 @@ sql insert into t4 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop13:
-sleep 100
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -534,7 +534,7 @@ sql insert into t1 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop14:
-sleep 100
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1

tests/script/tsim/stream/partitionbyColumn1.sim

@@ -24,11 +24,11 @@ sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop0:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -45,12 +45,14 @@ endi
 
 sql insert into t1 values(1648791213000,1,2,3,1.0);
 
+$loop_count = 0
+
 loop1:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -66,12 +68,14 @@ endi
 
 sql insert into t1 values(1648791213000,2,2,3,1.0);
 
+$loop_count = 0
+
 loop2:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -90,12 +94,14 @@ sql insert into t1 values(1648791213001,2,2,3,1.0);
 sql insert into t1 values(1648791213002,2,2,3,1.0);
 sql insert into t1 values(1648791213002,1,2,3,1.0);
 
+$loop_count = 0
+
 loop3:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -125,12 +131,14 @@ sql insert into t1 values(1648791223002,3,2,3,1.0);
 sql insert into t1 values(1648791223003,3,2,3,1.0);
 sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
 
+$loop_count = 0
+
 loop4:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -199,11 +207,11 @@ sql insert into t1 values(1648791213001,1,2,3,2.0);
 $loop_count = 0
 
 loop5:
-sleep 300
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -217,12 +225,14 @@ sql insert into t1 values(1648791223001,1,2,5,2.0);
 sql insert into t1 values(1648791223002,1,2,5,2.0);
 sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
 
+$loop_count = 0
+
 loop6:
-sleep 300
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -282,11 +292,11 @@ sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop7:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -303,12 +313,14 @@ endi
 sql insert into t1 values(1648791213000,1,2,3,1.0);
 sql insert into t2 values(1648791213000,1,2,3,1.0);
 
+$loop_count = 0
+
 loop8:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -324,12 +336,15 @@ endi
 
 sql insert into t1 values(1648791213000,2,2,3,1.0);
 sql insert into t2 values(1648791213000,2,2,3,1.0);
+
+$loop_count = 0
+
 loop9:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -352,12 +367,14 @@ sql insert into t2 values(1648791213001,2,2,3,1.0);
 sql insert into t2 values(1648791213002,2,2,3,1.0);
 sql insert into t2 values(1648791213002,1,2,3,1.0);
 
+$loop_count = 0
+
 loop10:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -373,7 +390,7 @@ endi
 
 if $data11 != 2 thenloop4
   print =====data11=$data11
-  goto loop3
+  goto loop10
 endi
 
 if $data12 != 1 then
@@ -392,12 +409,14 @@ sql insert into t2 values(1648791223002,3,2,3,1.0);
 sql insert into t2 values(1648791223003,3,2,3,1.0);
 sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
 
+$loop_count = 0
+
 loop11:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -470,17 +489,17 @@ sql insert into t4 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop13:
-sleep 300
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
 if $rows != 2 then
   print =====rows=$rows
-  goto loop14
+  goto loop13
 endi
 
 if $data01 != 1 then
@@ -495,12 +514,12 @@ endi
 
 if $data11 != 3 then
   print =====data11=$data11
-  goto loop11
+  goto loop13
 endi
 
 if $data12 != 2 then
   print =====data12=$data12
-  goto loop11
+  goto loop13
 endi
 
 sql insert into t4 values(1648791213000,2,2,3,1.0);
@@ -509,12 +528,14 @@ sql insert into t1 values(1648791233000,2,2,3,1.0);
 
 sql insert into t1 values(1648791213000,1,2,3,1.0); 
 
+$loop_count = 0
+
 loop14:
-sleep 300
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 

tests/script/tsim/stream/partitionbyColumn2.sim

@@ -40,6 +40,8 @@ endi
 
 sql insert into t1 values(1648791213000,1,1,3,1.0);
 
+$loop_count = 0
+
 loop1:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -61,6 +63,8 @@ endi
 
 sql insert into t1 values(1648791213000,2,1,3,1.0);
 
+$loop_count = 0
+
 loop2:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -85,6 +89,8 @@ sql insert into t1 values(1648791213001,2,1,3,1.0);
 sql insert into t1 values(1648791213002,2,1,3,1.0);
 sql insert into t1 values(1648791213002,1,1,3,1.0);
 
+$loop_count = 0
+
 loop3:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -120,6 +126,8 @@ sql insert into t1 values(1648791223002,3,2,3,1.0);
 sql insert into t1 values(1648791223003,3,2,3,1.0);
 sql insert into t1 values(1648791213001,1,1,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
 
+$loop_count = 0
+
 loop4:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -212,6 +220,8 @@ sql insert into t1 values(1648791223001,1,2,2,5);
 sql insert into t1 values(1648791223002,1,2,2,6);
 sql insert into t1 values(1648791213001,1,1,1,7) (1648791223002,1,1,2,8);
 
+$loop_count = 0
+
 loop6:
 sleep 300
 sql select * from streamt1 order by c1, c4, c2, c3;

utils/test/c/sml_test.c

@@ -92,7 +92,7 @@ int smlProcess_telnet_Test() {
 int smlProcess_json1_Test() {
   TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
 
-  TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1");
+  TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db");
   taos_free_result(pRes);
 
   pRes = taos_query(taos, "use sml_db");
@@ -112,7 +112,7 @@ int smlProcess_json1_Test() {
       "    },"
       "    {"
       "        \"metric\": \"sys.cpu.nice\","
-      "        \"timestamp\": 1346846400,"
+      "        \"timestamp\": 1662344042,"
       "        \"value\": 9,"
       "        \"tags\": {"
       "           \"host\": \"web02\","
@@ -141,7 +141,7 @@ int smlProcess_json2_Test() {
       "{"
       "    \"metric\": \"meter_current0\","
       "    \"timestamp\": {"
-      "        \"value\"  : 1346846400,"
+      "        \"value\"  : 1662344042,"
       "        \"type\"   : \"s\""
       "    },"
       "    \"value\": {"
@@ -181,7 +181,7 @@ int smlProcess_json3_Test() {
       "{"
       "    \"metric\": \"meter_current1\","
       "    \"timestamp\": {"
-      "        \"value\"  : 1346846400,"
+      "        \"value\"  : 1662344042,"
       "        \"type\"   : \"s\""
       "    },"
       "    \"value\": {"
@@ -249,7 +249,7 @@ int smlProcess_json4_Test() {
       "{"
       "    \"metric\": \"meter_current2\","
       "    \"timestamp\": {"
-      "        \"value\"  : 1346846500000,"
+      "        \"value\"  : 1662344042000,"
       "        \"type\"   : \"ms\""
       "    },"
       "    \"value\": \"ni\","