Merge pull request #20493 from taosdata/fix/TD-23172

ehn(tsdb/cache): load necessary columns only

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -745,6 +745,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
                                 int nCols) {
   SFSNextRowIter *state = (SFSNextRowIter *)iter;
   int32_t         code = 0;
+  bool            checkRemainingRow = true;
 
   switch (state->state) {
     case SFSNEXTROW_FS:
@@ -853,17 +854,60 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
         *pIgnoreEarlierTs = false;
         tBlockDataReset(state->pBlockData);
         TABLEID tid = {.suid = state->suid, .uid = state->uid};
-        code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0);
+        code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, aCols, nCols);
         if (code) goto _err;
 
         code = tsdbReadDataBlock(*state->pDataFReader, &block, state->pBlockData);
         if (code) goto _err;
 
+        for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) {
+          SColData *pColData = &state->pBlockData->aColData[colIndex];
+
+          if (isLast && (pColData->flag & HAS_VALUE)) {
+            skipBlock = false;
+            break;
+          } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+            skipBlock = false;
+            break;
+          }
+        }
+
+        if (skipBlock) {
+          if (--state->iBlock < 0) {
+            tsdbDataFReaderClose(state->pDataFReader);
+            *state->pDataFReader = NULL;
+            // resetLastBlockLoadInfo(state->pLoadInfo);
+
+            if (state->aBlockIdx) {
+              // taosArrayDestroy(state->aBlockIdx);
+              tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle);
+
+              state->aBlockIdxHandle = NULL;
+              state->aBlockIdx = NULL;
+            }
+
+            state->state = SFSNEXTROW_FILESET;
+            goto _next_fileset;
+          } else {
+            goto _next_datablock;
+          }
+        }
+
+        state->nRow = state->blockData.nRow;
+        state->iRow = state->nRow - 1;
+
+        state->state = SFSNEXTROW_BLOCKROW;
+        checkRemainingRow = false;
+      }
+    case SFSNEXTROW_BLOCKROW: {
+      if (checkRemainingRow) {
+        bool skipBlock = true;
+        int  inputColIndex = 0;
         for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) {
           SColData *pColData = &state->pBlockData->aColData[colIndex];
           int16_t   cid = pColData->cid;
 
-          if (inputColIndex < nCols && cid == aCols[inputColIndex++]) {
+          if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
             if (isLast && pColData->numOfValue != 0) {
               skipBlock = false;
               break;
@@ -871,6 +915,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
               skipBlock = false;
               break;
             }
+
+            ++inputColIndex;
           }
         }
 
@@ -894,13 +940,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
             goto _next_datablock;
           }
         }
-
-        state->nRow = state->blockData.nRow;
-        state->iRow = state->nRow - 1;
-
-        state->state = SFSNEXTROW_BLOCKROW;
       }
-    case SFSNEXTROW_BLOCKROW:
+
       if (state->iRow >= 0) {
         state->row = tsdbRowFromBlockData(state->pBlockData, state->iRow);
         *ppRow = &state->row;
@@ -926,6 +967,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
       }
 
       return code;
+    }
     default:
       ASSERT(0);
       break;
@@ -1545,6 +1587,9 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
 
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  for (int i = 1; i < pTSchema->numOfCols; ++i) {
+    taosArrayPush(aColArray, &pTSchema->columns[i].colId);
+  }
 
   TSKEY lastRowTs = TSKEY_MAX;
 
@@ -1600,11 +1645,13 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
         }
 
         if (!COL_VAL_IS_VALUE(pColVal)) {
-          taosArrayPush(aColArray, &pColVal->cid);
           if (!setNoneCol) {
             noneCol = iCol;
             setNoneCol = true;
           }
+        } else {
+          int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ);
+          taosArrayRemove(aColArray, aColIndex);
         }
       }
       if (!setNoneCol) {