Merge pull request #19486 from taosdata/fix/liaohj

enh(query): disable the memset when allocate the memory for dataBlock.

include/common/tdatablock.h

@@ -41,6 +41,12 @@ typedef struct SBlockOrderInfo {
     BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
   } while (0)
 
+#define colDataSetNull_f_s(c_, r_)                                        \
+  do {                                                                    \
+    colDataSetNull_f((c_)->nullbitmap, r_);                               \
+    memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \
+  } while (0)
+
 #define colDataClearNull_f(bm_, r_)                             \
   do {                                                          \
     BMCharPos(bm_, r_) &= ((char)(~(1u << (7u - BitPos(r_))))); \
@@ -136,7 +142,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     colDataSetNull_var(pColumnInfoData, currentRow);  // it is a null value of VAR type.
   } else {
-    colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow);
+    colDataSetNull_f_s(pColumnInfoData, currentRow);
   }
 
   pColumnInfoData->hasNull = true;
@@ -151,6 +157,7 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui
     for (int32_t i = start; i < start + nRows; ++i) {
       colDataSetNull_f(pColumnInfoData->nullbitmap, i);
     }
+    memset(pColumnInfoData->pData + start * pColumnInfoData->info.bytes, 0, pColumnInfoData->info.bytes * nRows);
   }
 
   pColumnInfoData->hasNull = true;
@@ -231,7 +238,6 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
 
 int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload);
 int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
-int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows);
 
 void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
 void blockDataCleanup(SSDataBlock* pDataBlock);

source/common/src/tdatablock.c

@@ -69,7 +69,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
     if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
       pColumnInfoData->varmeta.offset[currentRow] = -1;  // it is a null value of VAR type.
     } else {
-      colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow);
+      colDataSetNull_f_s(pColumnInfoData, currentRow);
     }
 
     pColumnInfoData->hasNull = true;
@@ -825,7 +825,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB
     } else {
       for (int32_t j = 0; j < pDataBlock->info.rows; ++j) {
         if (colDataIsNull_f(pSrc->nullbitmap, index[j])) {
-          colDataSetNull_f(pDst->nullbitmap, j);
+          colDataSetNull_f_s(pDst, j);
           continue;
         }
         memcpy(pDst->pData + j * pDst->info.bytes, pSrc->pData + index[j] * pDst->info.bytes, pDst->info.bytes);
@@ -1161,15 +1161,16 @@ void blockDataEmpty(SSDataBlock* pDataBlock) {
   pInfo->window.skey = 0;
 }
 
-// todo temporarily disable it
+/*
+ * NOTE: the type of the input column may be TSDB_DATA_TYPE_NULL, which is used to denote
+ * the all NULL value in this column. It is an internal representation of all NULL value column, and no visible to
+ * any users. The length of TSDB_DATA_TYPE_NULL is 0, and it is an special case.
+ */
 static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, bool clearPayload) {
   if (numOfRows <= 0 || numOfRows <= pBlockInfo->capacity) {
     return TSDB_CODE_SUCCESS;
   }
 
-  // todo temp disable it
-  //  ASSERT(pColumn->info.bytes != 0);
-
   int32_t existedRows = pBlockInfo->rows;
 
   if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
@@ -1194,7 +1195,8 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo*
       return TSDB_CODE_FAILED;
     }
 
-    // make sure the allocated memory is MALLOC_ALIGN_BYTES aligned
+    // here we employ the aligned malloc function, to make sure that the address of allocated memory is aligned
+    // to MALLOC_ALIGN_BYTES
     tmp = taosMemoryMallocAlign(MALLOC_ALIGN_BYTES, numOfRows * pColumn->info.bytes);
     if (tmp == NULL) {
       return TSDB_CODE_OUT_OF_MEMORY;
@@ -1208,7 +1210,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo*
 
     pColumn->pData = tmp;
 
-    // todo remove it soon
+    // check if the allocated memory is aligned to the requried bytes.
 #if defined LINUX
     if ((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) != 0x0) {
       return TSDB_CODE_FAILED;
@@ -1249,25 +1251,6 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
     return TSDB_CODE_SUCCESS;
   }
 
-  size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
-  for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, true);
-    if (code) {
-      return code;
-    }
-  }
-
-  pDataBlock->info.capacity = numOfRows;
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows) {
-  int32_t code = 0;
-  if (numOfRows == 0 || numOfRows <= pDataBlock->info.capacity) {
-    return TSDB_CODE_SUCCESS;
-  }
-
   size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);

source/libs/executor/src/projectoperator.c

@@ -279,7 +279,6 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       // for stream interval
       if (pBlock->info.type == STREAM_RETRIEVE || pBlock->info.type == STREAM_DELETE_RESULT ||
           pBlock->info.type == STREAM_DELETE_DATA) {
-        // printDataBlock1(pBlock, "project1");
         return pBlock;
       }
 

source/libs/executor/src/sysscanoperator.c

@@ -1918,6 +1918,13 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
   colDataAppend(pColInfo, 0, p, false);
   taosMemoryFree(p);
 
+  // make the valgrind happy that all memory buffer has been initialized already.
+  if (slotId != 0) {
+    SColumnInfoData* p1 = taosArrayGet(pBlock->pDataBlock, 0);
+    int64_t v = 0;
+    colDataAppendInt64(p1, 0, &v);
+  }
+
   pBlock->info.rows = 1;
   pOperator->status = OP_EXEC_DONE;
   return pBlock;

source/libs/function/src/builtinsimpl.c

@@ -1662,7 +1662,9 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) {
 
 int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   SVariant* pVal = &pCtx->param[1].param;
+  int32_t code = 0;
   double    v = 0;
+
   GET_TYPED_DATA(v, double, pVal->nType, &pVal->i);
 
   SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
@@ -1670,14 +1672,14 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
 
   tMemBucket* pMemBucket = ppInfo->pMemBucket;
   if (pMemBucket != NULL && pMemBucket->total > 0) {  // check for null
-    int32_t code = getPercentile(pMemBucket, v, &ppInfo->result);
-    if (code != TSDB_CODE_SUCCESS) {
-      tMemBucketDestroy(pMemBucket);
-      return code;
-    }
+    code = getPercentile(pMemBucket, v, &ppInfo->result);
   }
 
   tMemBucketDestroy(pMemBucket);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
   return functionFinalize(pCtx, pBlock);
 }
 
@@ -2644,7 +2646,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
       int32_t v = *(int32_t*)pv;
       int64_t delta = factor * (v - pDiffInfo->prev.i64);  // direct previous may be null
       if (delta < 0 && pDiffInfo->ignoreNegative) {
-        colDataSetNull_f(pOutput->nullbitmap, pos);
+        colDataSetNull_f_s(pOutput, pos);
       } else {
         colDataAppendInt64(pOutput, pos, &delta);
       }
@@ -2657,7 +2659,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
       int8_t  v = *(int8_t*)pv;
       int64_t delta = factor * (v - pDiffInfo->prev.i64);  // direct previous may be null
       if (delta < 0 && pDiffInfo->ignoreNegative) {
-        colDataSetNull_f(pOutput->nullbitmap, pos);
+        colDataSetNull_f_s(pOutput, pos);
       } else {
         colDataAppendInt64(pOutput, pos, &delta);
       }
@@ -2668,7 +2670,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
       int16_t v = *(int16_t*)pv;
       int64_t delta = factor * (v - pDiffInfo->prev.i64);  // direct previous may be null
       if (delta < 0 && pDiffInfo->ignoreNegative) {
-        colDataSetNull_f(pOutput->nullbitmap, pos);
+        colDataSetNull_f_s(pOutput, pos);
       } else {
         colDataAppendInt64(pOutput, pos, &delta);
       }
@@ -2680,7 +2682,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
       int64_t v = *(int64_t*)pv;
       int64_t delta = factor * (v - pDiffInfo->prev.i64);  // direct previous may be null
       if (delta < 0 && pDiffInfo->ignoreNegative) {
-        colDataSetNull_f(pOutput->nullbitmap, pos);
+        colDataSetNull_f_s(pOutput, pos);
       } else {
         colDataAppendInt64(pOutput, pos, &delta);
       }
@@ -2691,7 +2693,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
       float  v = *(float*)pv;
       double delta = factor * (v - pDiffInfo->prev.d64);                               // direct previous may be null
       if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) {  // check for overflow
-        colDataSetNull_f(pOutput->nullbitmap, pos);
+        colDataSetNull_f_s(pOutput, pos);
       } else {
         colDataAppendDouble(pOutput, pos, &delta);
       }
@@ -2702,7 +2704,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
       double v = *(double*)pv;
       double delta = factor * (v - pDiffInfo->prev.d64);                               // direct previous may be null
       if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) {  // check for overflow
-        colDataSetNull_f(pOutput->nullbitmap, pos);
+        colDataSetNull_f_s(pOutput, pos);
       } else {
         colDataAppendDouble(pOutput, pos, &delta);
       }
@@ -2737,7 +2739,7 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) {
 
       if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
         if (pDiffInfo->includeNull) {
-          colDataSetNull_f(pOutput->nullbitmap, pos);
+          colDataSetNull_f_s(pOutput, pos);
 
           numOfElems += 1;
         }
@@ -2775,8 +2777,7 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) {
 
       if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
         if (pDiffInfo->includeNull) {
-          colDataSetNull_f(pOutput->nullbitmap, pos);
-
+          colDataSetNull_f_s(pOutput, pos);
           numOfElems += 1;
         }
         continue;

source/libs/function/src/tpercentile.c

@@ -92,6 +92,7 @@ static void resetPosInfo(SSlotInfo *pInfo) {
 
 int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) {
   ASSERT(pMemBucket->total == 1);
+  terrno = 0;
 
   for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) {
     tMemBucketSlot *pSlot = &pMemBucket->pSlots[i];

source/libs/scalar/src/filter.c

@@ -3181,6 +3181,7 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, SColumnInfoData *pRe
     void *colData = colDataGetData(pData, i);
     if (colData == NULL || colDataIsNull_s(pData, i)) {
       all = false;
+      p[i] = 0;
       continue;
     }
 

tests/script/tsim/parser/alter1.sim

@@ -88,6 +88,7 @@ sql insert into car1 values (now, 1, 1,1 ) (now +1s, 2,2,2) car2 values (now, 1,
 
 sql select c1+speed from stb where c1 > 0
 if $rows != 3 then
+  print $rows , expect 3
   return -1
 endi