refactor: do some internal refactor.

include/common/tdataformat.h

@@ -130,7 +130,7 @@ void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn)
 void    tColDataClear(SColData *pColData);
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
 void    tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
-uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal);
+uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal);
 int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
 extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull);
 

source/common/src/tdataformat.c

@@ -1557,7 +1557,7 @@ void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
   tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal);
 }
 
-uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal) {
+uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) {
   uint8_t v;
   switch (pColData->flag) {
     case HAS_NONE:

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -910,7 +910,7 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData
   return endPos;
 }
 
-static void copyPrimaryTsCol(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData,
+static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData,
                              int32_t dumpedRows, bool asc) {
   if (asc) {
     memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t));
@@ -930,74 +930,9 @@ static void copyPrimaryTsCol(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpIn
   }
 }
 
-static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) {
-  SReaderStatus*  pStatus = &pReader->status;
-  SDataBlockIter* pBlockIter = &pStatus->blockIter;
-
-  SBlockData*         pBlockData = &pStatus->fileBlockData;
-  SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter);
-  SDataBlk*           pBlock = getCurrentBlock(pBlockIter);
-  SSDataBlock*        pResBlock = pReader->pResBlock;
-  int32_t             numOfOutputCols = blockDataGetNumOfCols(pResBlock);
-
-  SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
-  SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
-
-  SColVal cv = {0};
-  int64_t st = taosGetTimestampUs();
-  bool    asc = ASCENDING_TRAVERSE(pReader->order);
-  int32_t step = asc ? 1 : -1;
-
-  if ((pDumpInfo->rowIndex == 0 && asc) || (pDumpInfo->rowIndex == pBlock->nRow - 1 && (!asc))) {
-    if (asc && pReader->window.skey <= pBlock->minKey.ts) {
-      // pDumpInfo->rowIndex = 0;
-    } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts) {
-      // pDumpInfo->rowIndex = pBlock->nRow - 1;
-    } else {
-      int32_t pos = asc ? pBlock->nRow - 1 : 0;
-      int32_t order = asc ? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
-      int64_t key = asc ? pReader->window.skey : pReader->window.ekey;
-      pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, order);
-    }
-  }
-
-  // time window check
-  int32_t endIndex = getEndPosInDataBlock(pReader, pBlockData, pBlock, pDumpInfo->rowIndex);
-  if (endIndex == -1) {
-    setBlockAllDumped(pDumpInfo, pReader->window.ekey, pReader->order);
-    return TSDB_CODE_SUCCESS;
-  }
-
-  endIndex += step;
-  int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex);
-  if (dumpedRows > pReader->capacity) {  // output buffer check
-    dumpedRows = pReader->capacity;
-  }
-
-  int32_t i = 0;
-  int32_t rowIndex = 0;
-
-  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i);
-  if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-    copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);
-    i += 1;
-  }
-
-  int32_t colIndex = 0;
-  int32_t num = pBlockData->nColData;
-  while (i < numOfOutputCols && colIndex < num) {
-    rowIndex = 0;
-    pColData = taosArrayGet(pResBlock->pDataBlock, i);
-
-    SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);
-    if (pData->cid < pColData->info.colId) {
-      colIndex += 1;
-    } else if (pData->cid == pColData->info.colId) {
-      if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {
-        colDataAppendNNULL(pColData, 0, dumpedRows);
-      } else {
-        if (IS_MATHABLE_TYPE(pColData->info.type)) {
-
+// a faster version of copy procedure.
+static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData,
+                            int32_t dumpedRows, bool asc)  {
   uint8_t* p = NULL;
   if (asc) {
     p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex;
@@ -1006,13 +941,19 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
     p = pData->pData + tDataTypes[pData->type].bytes * startIndex;
   }
 
+  int32_t step = asc? 1:-1;
+
   // make sure it is aligned to 8bit
   ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0);
+
+  // 1. copy data in a batch model
   memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes);
 
+  // 2. reverse the array list in case of descending order scan data block
   if (!asc) {
     switch(pColData->info.type) {
       case TSDB_DATA_TYPE_TIMESTAMP:
+      case TSDB_DATA_TYPE_DOUBLE:
       case TSDB_DATA_TYPE_BIGINT:
       case TSDB_DATA_TYPE_UBIGINT:
       {
@@ -1051,6 +992,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
         break;
       }
 
+      case TSDB_DATA_TYPE_FLOAT:
       case TSDB_DATA_TYPE_INT:
       case TSDB_DATA_TYPE_UINT: {
         int32_t  mid = dumpedRows >> 1u;
@@ -1065,16 +1007,89 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
     }
   }
 
-          // null value exists, check one-by-one
+  // 3. if the  null value exists, check items one-by-one
   if (pData->flag != HAS_VALUE) {
+    int32_t rowIndex = 0;
+
     for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step, rowIndex++) {
       uint8_t v = tColDataGetBitValue(pData, j);
       if (v == 0 || v == 1) {
         colDataSetNull_f(pColData->nullbitmap, rowIndex);
+        pColData->hasNull = true;
+      }
+    }
+  }
+}
+
+static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) {
+  SReaderStatus*  pStatus = &pReader->status;
+  SDataBlockIter* pBlockIter = &pStatus->blockIter;
+
+  SBlockData*         pBlockData = &pStatus->fileBlockData;
+  SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter);
+  SDataBlk*           pBlock = getCurrentBlock(pBlockIter);
+  SSDataBlock*        pResBlock = pReader->pResBlock;
+  int32_t             numOfOutputCols = blockDataGetNumOfCols(pResBlock);
+
+  SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
+  SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
+
+  SColVal cv = {0};
+  int64_t st = taosGetTimestampUs();
+  bool    asc = ASCENDING_TRAVERSE(pReader->order);
+  int32_t step = asc ? 1 : -1;
+
+  if ((pDumpInfo->rowIndex == 0 && asc) || (pDumpInfo->rowIndex == pBlock->nRow - 1 && (!asc))) {
+    if (asc && pReader->window.skey <= pBlock->minKey.ts) {
+      // pDumpInfo->rowIndex = 0;
+    } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts) {
+      // pDumpInfo->rowIndex = pBlock->nRow - 1;
+    } else {
+      int32_t pos = asc ? pBlock->nRow - 1 : 0;
+      int32_t order = asc ? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
+      int64_t key = asc ? pReader->window.skey : pReader->window.ekey;
+      pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, order);
     }
   }
+
+  // time window check
+  int32_t endIndex = getEndPosInDataBlock(pReader, pBlockData, pBlock, pDumpInfo->rowIndex);
+  if (endIndex == -1) {
+    setBlockAllDumped(pDumpInfo, pReader->window.ekey, pReader->order);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  endIndex += step;
+  int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex);
+  if (dumpedRows > pReader->capacity) {  // output buffer check
+    dumpedRows = pReader->capacity;
   }
+
+  int32_t i = 0;
+  int32_t rowIndex = 0;
+
+  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i);
+  if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+    copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);
+    i += 1;
+  }
+
+  int32_t colIndex = 0;
+  int32_t num = pBlockData->nColData;
+  while (i < numOfOutputCols && colIndex < num) {
+    rowIndex = 0;
+    pColData = taosArrayGet(pResBlock->pDataBlock, i);
+
+    SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);
+    if (pData->cid < pColData->info.colId) {
+      colIndex += 1;
+    } else if (pData->cid == pColData->info.colId) {
+      if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {
+        colDataAppendNNULL(pColData, 0, dumpedRows);
       } else {
+        if (IS_MATHABLE_TYPE(pColData->info.type)) {
+          copyNumericCols(pData, pDumpInfo, pColData, dumpedRows, asc);
+        } else {  // varchar/nchar type
           for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) {
             tColDataGetValue(pData, j, &cv);
             doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo);

source/libs/function/src/builtinsimpl.c

@@ -3022,6 +3022,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
     }
   }
 #endif
+
   if (numOfElems == 0) {
     // save selectivity value for column consisted of all null values
     firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo);