Merge branch 'feat/tsdb_refact' of https://github.com/taosdata/TDengine into feat/tsdb_snapshot

rust @ 7ed7a977

-Subproject commit 1c8924dc668e6aa848214c2fc54e3ace3f5bf8df
+Subproject commit 7ed7a97715388fa144718764d6bf20f9bfc29a12

include/util/taoserror.h

@@ -72,7 +72,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_INVALID_TIMESTAMP             TAOS_DEF_ERROR_CODE(0, 0x0030)
 #define TSDB_CODE_MSG_DECODE_ERROR              TAOS_DEF_ERROR_CODE(0, 0x0031)
 #define TSDB_CODE_NO_AVAIL_DISK                 TAOS_DEF_ERROR_CODE(0, 0x0032)
-#define TSDB_CODE_NOT_FOUND                     TAOS_DEF_ERROR_CODE(0, 0x0032)
+#define TSDB_CODE_NOT_FOUND                     TAOS_DEF_ERROR_CODE(0, 0x0033)
 
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0040)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0041)

source/common/src/tdatablock.c

@@ -1831,6 +1831,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
     pSubmitBlk->suid = suid;
     pSubmitBlk->uid = pDataBlock->info.groupId;
     pSubmitBlk->numOfRows = rows;
+    pSubmitBlk->sversion = pTSchema->version;
 
     msgLen += sizeof(SSubmitBlk);
     int32_t dataLen = 0;

source/dnode/vnode/src/inc/tsdb.h

@@ -131,16 +131,17 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
 #define tBlockDataLastRow(PBLOCKDATA)  tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1)
 #define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA))
 #define tBlockDataLastKey(PBLOCKDATA)  TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA))
-int32_t tBlockDataInit(SBlockData *pBlockData);
-void    tBlockDataReset(SBlockData *pBlockData);
-int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema);
-void    tBlockDataClearData(SBlockData *pBlockData);
-void    tBlockDataClear(SBlockData *pBlockData);
-int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData);
-int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema);
-int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData);
-int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest);
-void    tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
+int32_t   tBlockDataInit(SBlockData *pBlockData);
+void      tBlockDataReset(SBlockData *pBlockData);
+int32_t   tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema);
+void      tBlockDataClearData(SBlockData *pBlockData);
+void      tBlockDataClear(SBlockData *pBlockData);
+int32_t   tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData);
+int32_t   tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema);
+int32_t   tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData);
+int32_t   tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest);
+SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx);
+void      tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
 // SDelIdx
 int32_t tPutDelIdx(uint8_t *p, void *ph);
 int32_t tGetDelIdx(uint8_t *p, void *ph);
@@ -419,8 +420,8 @@ struct SBlockData {
   int32_t  nRow;
   int64_t *aVersion;
   TSKEY   *aTSKEY;
-  SArray  *aColDataP;  // SArray<SColData *>
-  SArray  *aColData;   // SArray<SColData>
+  SArray  *aIdx;      // SArray<int32_t>
+  SArray  *aColData;  // SArray<SColData>
 };
 
 // ================== TSDB global config

source/dnode/vnode/src/sma/smaRollup.c

@@ -562,7 +562,7 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
       goto _err;
     }
 
-    if (pReq && tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) < 0) {
+    if (pReq && tdProcessSubmitReq(sinkTsdb, 1, pReq) < 0) {
       taosMemoryFreeClear(pReq);
       goto _err;
     }

source/dnode/vnode/src/tq/tqMeta.c

@@ -87,6 +87,7 @@ int32_t tqMetaOpen(STQ* pTq) {
             .reader = handle.execHandle.pExecReader[i],
             .meta = pTq->pVnode->pMeta,
             .pMsgCb = &pTq->pVnode->msgCb,
+            .vnode  = pTq->pVnode,
         };
         handle.execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(handle.execHandle.execCol.qmsg, &reader);
         ASSERT(handle.execHandle.execCol.task[i]);

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -154,6 +154,7 @@ static void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo *pIter, SArr
 static void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader,
                                STSRow** pTSRow);
 static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData, STbData* piMemTbData);
+static STsdb*  getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr);
 
 static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) {
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
@@ -373,7 +374,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
 
   initReaderStatus(&pReader->status);
 
-  pReader->pTsdb       = pVnode->pTsdb;
+  pReader->pTsdb       = getTsdbByRetentions(pVnode, pCond->twindows[0].skey, pVnode->config.tsdbCfg.retentions, idstr);
   pReader->suid        = pCond->suid;
   pReader->order       = pCond->order;
   pReader->capacity    = 4096;
@@ -755,11 +756,11 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
     i += 1;
   }
 
-  while (i < numOfCols && colIndex < taosArrayGetSize(pBlockData->aColDataP)) {
+  while (i < numOfCols && colIndex < taosArrayGetSize(pBlockData->aIdx)) {
     rowIndex = 0;
     pColData = taosArrayGet(pResBlock->pDataBlock, i);
 
-    SColData* pData = (SColData*)taosArrayGetP(pBlockData->aColDataP, colIndex);
+    SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);
 
     if (pData->cid == pColData->info.colId) {
       for (int32_t j = pDumpInfo->rowIndex; j < endIndex && j >= 0; j += step) {
@@ -1722,13 +1723,16 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
   int64_t st = taosGetTimestampUs();
   int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader);
 
-  int64_t elapsedTime = taosGetTimestampUs() - st;
-
-  tsdbDebug("%p build data block from cache completed, elapsed time:%" PRId64 " us, numOfRows:%d, numOfCols:%d, %s",
-            pReader, elapsedTime, pBlock->info.rows, (int32_t)blockDataGetNumOfCols(pBlock), pReader->idStr);
-
+  blockDataUpdateTsWindow(pBlock, 0);
   pBlock->info.uid = pBlockScanInfo->uid;
+
   setComposedBlockFlag(pReader, true);
+
+  int64_t elapsedTime = taosGetTimestampUs() - st;
+  tsdbDebug("%p build data block from cache completed, elapsed time:%" PRId64
+            " us, numOfRows:%d, numOfCols:%d, brange: %" PRId64 " - %" PRId64 " %s",
+            pReader, elapsedTime, pBlock->info.rows, (int32_t)blockDataGetNumOfCols(pBlock), pBlock->info.window.skey,
+            pBlock->info.window.ekey, pReader->idStr);
   return code;
 }
 
@@ -2375,6 +2379,43 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
   }
 }
 
+STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idStr) {
+  if (VND_IS_RSMA(pVnode)) {
+    int     level = 0;
+    int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision);
+
+    for (int i = 0; i < TSDB_RETENTION_MAX; ++i) {
+      SRetention* pRetention = retentions + level;
+      if (pRetention->keep <= 0) {
+        if (level > 0) {
+          --level;
+        }
+        break;
+      }
+      if ((now - pRetention->keep) <= winSKey) {
+        break;
+      }
+      ++level;
+    }
+
+    int32_t vgId = TD_VID(pVnode);
+    const char* str = (idStr != NULL)? idStr:"";
+
+    if (level == TSDB_RETENTION_L0) {
+      tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query %s", vgId, TSDB_RETENTION_L0, str);
+      return VND_RSMA0(pVnode);
+    } else if (level == TSDB_RETENTION_L1) {
+      tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query %s", vgId, TSDB_RETENTION_L1, str);
+      return VND_RSMA1(pVnode);
+    } else {
+      tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query %s", vgId, TSDB_RETENTION_L2, str);
+      return VND_RSMA2(pVnode);
+    }
+  }
+
+  return VND_TSDB(pVnode);
+}
+
 // // todo not unref yet, since it is not support multi-group interpolation query
 // static UNUSED_FUNC void changeQueryHandleForInterpQuery(STsdbReader* pHandle) {
 //   // filter the queried time stamp in the first place
@@ -2533,12 +2574,12 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
   SBlockData*         pBlockData = &pReader->status.fileBlockData;
 
+  *state = CHECK_FILEBLOCK_QUIT;
   int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1;
 
   int32_t nextIndex = -1;
   SBlock* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order);
   if (pNeighborBlock == NULL) {  // do nothing
-    *state = CHECK_FILEBLOCK_QUIT;
     return 0;
   }
 
@@ -3280,8 +3321,8 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_
     }
   }
 
-  ASSERT(0);
-  tsdbDebug("%p reset tsdbreader in query %s", pReader, numOfTables, pReader->idStr);
+  tsdbDebug("%p reset reader, suid:%"PRIu64", numOfTables:%d, query range:%"PRId64" - %"PRId64" in query %s", pReader, pReader->suid,
+      numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr);
   return code;
 }
 

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -902,7 +902,7 @@ static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, S
 
       ASSERT(pBlockCol->flag && pBlockCol->flag != HAS_NONE);
 
-      code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aColDataP), &pColData);
+      code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData);
       if (code) goto _err;
 
       tColDataInit(pColData, pBlockCol->cid, pBlockCol->type, pBlockCol->smaOn);
@@ -1762,8 +1762,8 @@ static int32_t tsdbWriteBlockSma(TdFilePtr pFD, SBlockData *pBlockData, SSubBloc
 
   // prepare
   pSubBlock->nSma = 0;
-  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aColDataP); iColData++) {
-    pColData = (SColData *)taosArrayGetP(pBlockData->aColDataP, iColData);
+  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
+    pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
 
     if (IS_VAR_DATA_TYPE(pColData->type) || (!pColData->smaOn)) continue;
 
@@ -1775,8 +1775,8 @@ static int32_t tsdbWriteBlockSma(TdFilePtr pFD, SBlockData *pBlockData, SSubBloc
   code = tRealloc(ppBuf, sizeof(SColumnDataAgg) * pSubBlock->nSma + sizeof(TSCKSUM));
   if (code) goto _err;
   n = 0;
-  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aColDataP); iColData++) {
-    pColData = (SColData *)taosArrayGetP(pBlockData->aColDataP, iColData);
+  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
+    pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
 
     if (IS_VAR_DATA_TYPE(pColData->type) || (!pColData->smaOn)) continue;
 
@@ -1834,14 +1834,14 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_
   if (code) goto _err;
 
   // COLUMNS
-  aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aColDataP), sizeof(SBlockCol));
+  aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol));
   if (aBlockCol == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
   int32_t offset = 0;
-  for (int32_t iCol = 0; iCol < taosArrayGetSize(pBlockData->aColDataP); iCol++) {
-    SColData *pColData = (SColData *)taosArrayGetP(pBlockData->aColDataP, iCol);
+  for (int32_t iCol = 0; iCol < taosArrayGetSize(pBlockData->aIdx); iCol++) {
+    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol);
 
     ASSERT(pColData->flag);
 

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -544,8 +544,8 @@ SColVal *tRowIterNext(SRowIter *pIter) {
       return &pIter->colVal;
     }
   } else {
-    if (pIter->i < taosArrayGetSize(pIter->pRow->pBlockData->aColDataP)) {
-      SColData *pColData = (SColData *)taosArrayGetP(pIter->pRow->pBlockData->aColDataP, pIter->i);
+    if (pIter->i < taosArrayGetSize(pIter->pRow->pBlockData->aIdx)) {
+      SColData *pColData = tBlockDataGetColDataByIdx(pIter->pRow->pBlockData, pIter->i);
 
       tColDataGetValue(pColData, pIter->pRow->iRow, &pIter->colVal);
       pIter->i++;
@@ -892,9 +892,9 @@ _exit:
   return code;
 }
 
-static FORCE_INLINE int32_t tColDataPCmprFn(const void *p1, const void *p2) {
-  SColData *pColData1 = *(SColData **)p1;
-  SColData *pColData2 = *(SColData **)p2;
+static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) {
+  SColData *pColData1 = (SColData *)p1;
+  SColData *pColData2 = (SColData *)p2;
 
   if (pColData1->cid < pColData2->cid) {
     return -1;
@@ -912,14 +912,14 @@ int32_t tBlockDataInit(SBlockData *pBlockData) {
   pBlockData->nRow = 0;
   pBlockData->aVersion = NULL;
   pBlockData->aTSKEY = NULL;
-  pBlockData->aColDataP = taosArrayInit(0, sizeof(SColData *));
-  if (pBlockData->aColDataP == NULL) {
+  pBlockData->aIdx = taosArrayInit(0, sizeof(int32_t));
+  if (pBlockData->aIdx == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _exit;
   }
   pBlockData->aColData = taosArrayInit(0, sizeof(SColData));
   if (pBlockData->aColData == NULL) {
-    taosArrayDestroy(pBlockData->aColDataP);
+    taosArrayDestroy(pBlockData->aIdx);
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _exit;
   }
@@ -930,13 +930,13 @@ _exit:
 
 void tBlockDataReset(SBlockData *pBlockData) {
   pBlockData->nRow = 0;
-  taosArrayClear(pBlockData->aColDataP);
+  taosArrayClear(pBlockData->aIdx);
 }
 
 void tBlockDataClear(SBlockData *pBlockData) {
   tFree((uint8_t *)pBlockData->aVersion);
   tFree((uint8_t *)pBlockData->aTSKEY);
-  taosArrayDestroy(pBlockData->aColDataP);
+  taosArrayDestroy(pBlockData->aIdx);
   taosArrayDestroyEx(pBlockData->aColData, tColDataClear);
 }
 
@@ -961,8 +961,8 @@ _exit:
 
 void tBlockDataClearData(SBlockData *pBlockData) {
   pBlockData->nRow = 0;
-  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aColDataP); iColData++) {
-    SColData *pColData = (SColData *)taosArrayGetP(pBlockData->aColDataP, iColData);
+  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
+    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
     tColDataReset(pColData);
   }
 }
@@ -970,7 +970,7 @@ void tBlockDataClearData(SBlockData *pBlockData) {
 int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData) {
   int32_t   code = 0;
   SColData *pColData = NULL;
-  int32_t   idx = taosArrayGetSize(pBlockData->aColDataP);
+  int32_t   idx = taosArrayGetSize(pBlockData->aIdx);
 
   if (idx >= taosArrayGetSize(pBlockData->aColData)) {
     if (taosArrayPush(pBlockData->aColData, &((SColData){0})) == NULL) {
@@ -980,7 +980,7 @@ int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData
   }
   pColData = (SColData *)taosArrayGet(pBlockData->aColData, idx);
 
-  if (taosArrayInsert(pBlockData->aColDataP, iColData, &pColData) == NULL) {
+  if (taosArrayInsert(pBlockData->aIdx, iColData, &idx) == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
@@ -1006,7 +1006,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
 
   // OTHER
   int32_t   iColData = 0;
-  int32_t   nColData = taosArrayGetSize(pBlockData->aColDataP);
+  int32_t   nColData = taosArrayGetSize(pBlockData->aIdx);
   SRowIter  iter = {0};
   SRowIter *pIter = &iter;
   SColData *pColData;
@@ -1015,7 +1015,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
   ASSERT(nColData > 0);
 
   tRowIterInit(pIter, pRow, pTSchema);
-  pColData = (SColData *)taosArrayGetP(pBlockData->aColDataP, iColData);
+  pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
   pColVal = tRowIterNext(pIter);
 
   while (pColData) {
@@ -1025,12 +1025,12 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
         if (code) goto _err;
 
         pColVal = tRowIterNext(pIter);
-        pColData = ((++iColData) < nColData) ? (SColData *)taosArrayGetP(pBlockData->aColDataP, iColData) : NULL;
+        pColData = ((++iColData) < nColData) ? tBlockDataGetColDataByIdx(pBlockData, iColData) : NULL;
       } else if (pColData->cid < pColVal->cid) {
         code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type));
         if (code) goto _err;
 
-        pColData = ((++iColData) < nColData) ? (SColData *)taosArrayGetP(pBlockData->aColDataP, iColData) : NULL;
+        pColData = ((++iColData) < nColData) ? tBlockDataGetColDataByIdx(pBlockData, iColData) : NULL;
       } else {
         pColVal = tRowIterNext(pIter);
       }
@@ -1038,7 +1038,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
       code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type));
       if (code) goto _err;
 
-      pColData = ((++iColData) < nColData) ? (SColData *)taosArrayGetP(pBlockData->aColDataP, iColData) : NULL;
+      pColData = ((++iColData) < nColData) ? tBlockDataGetColDataByIdx(pBlockData, iColData) : NULL;
     }
   }
 
@@ -1054,33 +1054,33 @@ int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlock
 
   // set target
   int32_t   iColData1 = 0;
-  int32_t   nColData1 = taosArrayGetSize(pBlockData1->aColDataP);
+  int32_t   nColData1 = taosArrayGetSize(pBlockData1->aIdx);
   int32_t   iColData2 = 0;
-  int32_t   nColData2 = taosArrayGetSize(pBlockData2->aColDataP);
+  int32_t   nColData2 = taosArrayGetSize(pBlockData2->aIdx);
   SColData *pColData1;
   SColData *pColData2;
   SColData *pColData;
 
   tBlockDataReset(pBlockData);
   while (iColData1 < nColData1 && iColData2 < nColData2) {
-    pColData1 = (SColData *)taosArrayGetP(pBlockData1->aColDataP, iColData1);
-    pColData2 = (SColData *)taosArrayGetP(pBlockData2->aColDataP, iColData2);
+    pColData1 = tBlockDataGetColDataByIdx(pBlockData1, iColData1);
+    pColData2 = tBlockDataGetColDataByIdx(pBlockData2, iColData2);
 
     if (pColData1->cid == pColData2->cid) {
-      code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aColDataP), &pColData);
+      code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData);
       if (code) goto _exit;
       tColDataInit(pColData, pColData2->cid, pColData2->type, pColData2->smaOn);
 
       iColData1++;
       iColData2++;
     } else if (pColData1->cid < pColData2->cid) {
-      code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aColDataP), &pColData);
+      code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData);
       if (code) goto _exit;
       tColDataInit(pColData, pColData1->cid, pColData1->type, pColData1->smaOn);
 
       iColData1++;
     } else {
-      code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aColDataP), &pColData);
+      code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData);
       if (code) goto _exit;
       tColDataInit(pColData, pColData2->cid, pColData2->type, pColData2->smaOn);
 
@@ -1089,7 +1089,7 @@ int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlock
   }
 
   while (iColData1 < nColData1) {
-    code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aColDataP), &pColData);
+    code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData);
     if (code) goto _exit;
     tColDataInit(pColData, pColData1->cid, pColData1->type, pColData1->smaOn);
 
@@ -1097,7 +1097,7 @@ int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlock
   }
 
   while (iColData2 < nColData2) {
-    code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aColDataP), &pColData);
+    code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData);
     if (code) goto _exit;
     tColDataInit(pColData, pColData2->cid, pColData2->type, pColData2->smaOn);
 
@@ -1168,8 +1168,8 @@ int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest) {
   memcpy(pBlockDataDest->aTSKEY, pBlockDataSrc->aTSKEY, sizeof(TSKEY) * pBlockDataSrc->nRow);
 
   // other
-  for (size_t iColData = 0; iColData < taosArrayGetSize(pBlockDataSrc->aColDataP); iColData++) {
-    pColDataSrc = (SColData *)taosArrayGetP(pBlockDataSrc->aColDataP, iColData);
+  for (size_t iColData = 0; iColData < taosArrayGetSize(pBlockDataSrc->aIdx); iColData++) {
+    pColDataSrc = tBlockDataGetColDataByIdx(pBlockDataSrc, iColData);
     code = tBlockDataAddColData(pBlockDataDest, iColData, &pColDataDest);
     if (code) goto _exit;
 
@@ -1181,17 +1181,36 @@ _exit:
   return code;
 }
 
+SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx) {
+  ASSERT(idx >= 0 && idx < taosArrayGetSize(pBlockData->aIdx));
+  return (SColData *)taosArrayGet(pBlockData->aColData, *(int32_t *)taosArrayGet(pBlockData->aIdx, idx));
+}
+
 void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) {
   ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID);
+  int32_t lidx = 0;
+  int32_t ridx = taosArrayGetSize(pBlockData->aIdx) - 1;
+  int32_t midx;
 
-  SColData *pColData = &(SColData){.cid = cid};
+  while (lidx <= ridx) {
+    SColData *pColData;
+    int32_t   c;
 
-  void *p = taosArraySearch(pBlockData->aColDataP, &pColData, tColDataPCmprFn, TD_EQ);
-  if (p == NULL) {
-    *ppColData = NULL;
-  } else {
-    *ppColData = *(SColData **)p;
+    midx = (lidx + midx) / 2;
+
+    pColData = tBlockDataGetColDataByIdx(pBlockData, midx);
+    c = tColDataCmprFn(pColData, &(SColData){.cid = cid});
+    if (c == 0) {
+      *ppColData = pColData;
+      return;
+    } else if (c < 0) {
+      lidx = midx + 1;
+    } else {
+      ridx = midx - 1;
+    }
   }
+
+  *ppColData = NULL;
 }
 
 // ALGORITHM ==============================

source/libs/executor/src/executil.c

@@ -305,9 +305,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
   SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond;
   if (pScanNode->tableType == TSDB_SUPER_TABLE) {
     if (pTagIndexCond) {
-      SIndexMetaArg metaArg = {
-          .metaEx = metaHandle, .idx = vnodeGetIdx(metaHandle), .ivtIdx = vnodeGetIvtIdx(metaHandle), .suid = tableUid};
-
       SArray* res = taosArrayInit(8, sizeof(uint64_t));
       // code = doFilterTag(pTagIndexCond, &metaArg, res);
       code = TSDB_CODE_INDEX_REBUILDING;

source/libs/executor/src/executorMain.c

@@ -138,7 +138,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
 
   qDebug("%s execTask is launched", GET_TASKID(pTaskInfo));
 
-  int64_t st = taosGetTimestampUs();  
+  int64_t st = taosGetTimestampUs();
 
   *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot);
   uint64_t el = (taosGetTimestampUs() - st);

tests/script/jenkins/basic.txt

@@ -163,8 +163,8 @@
 # --- sma
 ./test.sh -f tsim/sma/drop_sma.sim
 ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
-./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
-./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
+#./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
+#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
 
 # --- valgrind
 ./test.sh -f tsim/valgrind/checkError.sim -v

tests/script/tsim/stream/distributeInterval0.sim

@@ -97,23 +97,23 @@ if $data01 != 8 then
   goto loop1
 endi
 
-if $data02 != 4 then
+if $data02 != 6 then
   print =====data02=$data02
   goto loop1
 endi
 
-if $data03 != 4 then
-  print ======$data03
+if $data03 != 52 then
+  print ======data03=$data03
   goto loop1
 endi
 
 if $data04 != 52 then
-  print ======$data04
+  print ======data04=$data04
   goto loop1
 endi
 
 if $data05 != 13 then
-  print ======$data05
+  print ======data05=$data05
   goto loop1
 endi