[td-13039] fix bug in sorted merge operator.

include/common/tep.h

@@ -64,15 +64,15 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
   }
 }
 
-#define colDataGet(p1_, r_)                                                          \
+#define colDataGetData(p1_, r_)                                                          \
   ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? (p1_)->pData + (p1_)->varmeta.offset[(r_)] \
-                                        : (p1_)->pData + ((r_) * (p1_)->info.bytes));
+                                        : (p1_)->pData + ((r_) * (p1_)->info.bytes))
 
 int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
 int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2);
 int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock);
 
-int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
+int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
 void    colDataTrim(SColumnInfoData* pColumnInfoData);
 
 size_t colDataGetNumOfCols(const SSDataBlock* pBlock);
@@ -92,13 +92,13 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
 
 SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols);
 
-size_t blockDataNumOfRowsForSerialize(const SSDataBlock* pBlock, int32_t blockSize);
-
 int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
 int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
 
 int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
 void    blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol);
+SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock);
+size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
 void   *blockDataDestroy(SSDataBlock *pBlock);
 
 #ifdef __cplusplus

include/libs/function/function.h

@@ -138,8 +138,10 @@ extern SFunctionFpSet fpSet[1];
 
 // sql function runtime context
 typedef struct SqlFunctionCtx {
+  int32_t      startRow;
   int32_t      size;      // number of rows
-  void *       pInput;    // input data buffer
+  SColumnInfoData* pInput;
+
   uint32_t     order;     // asc|desc
   int16_t      inputType;
   int16_t      inputBytes;

source/common/src/tep.c

@@ -63,7 +63,7 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet) {
 
 #define BitmapLen(_n)     (((_n) + ((1<<NBIT)-1)) >> NBIT)
 
-int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) {
+int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) {
   ASSERT(pColumnInfoData != NULL);
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     return pColumnInfoData->varmeta.length;
@@ -249,8 +249,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) {
   }
 
   ASSERT(pColInfoData->nullbitmap == NULL);
-  pDataBlock->info.window.skey = *(TSKEY*) colDataGet(pColInfoData, 0);
-  pDataBlock->info.window.ekey = *(TSKEY*) colDataGet(pColInfoData, (pDataBlock->info.rows - 1));
+  pDataBlock->info.window.skey = *(TSKEY*) colDataGetData(pColInfoData, 0);
+  pDataBlock->info.window.ekey = *(TSKEY*) colDataGetData(pColInfoData, (pDataBlock->info.rows - 1));
   return 0;
 }
 
@@ -262,8 +262,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
     SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
     SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
 
-    uint32_t oldLen = colDataGetSize(pCol2, pDest->info.rows);
-    uint32_t newLen = colDataGetSize(pCol1, pSrc->info.rows);
+    uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows);
+    uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows);
 
     int32_t newSize = oldLen + newLen;
     char* tmp = realloc(pCol2->pData, newSize);
@@ -287,7 +287,7 @@ size_t blockDataGetSize(const SSDataBlock* pBlock) {
 
   for(int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-    total += colDataGetSize(pColInfoData, pBlock->info.rows);
+    total += colDataGetLength(pColInfoData, pBlock->info.rows);
 
     if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
       total += sizeof(int32_t) * pBlock->info.rows;
@@ -336,7 +336,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
           if (isNull) {
             // do nothing
           } else {
-            char* p = colDataGet(pColInfoData, j);
+            char* p = colDataGetData(pColInfoData, j);
             size += varDataTLen(p);
           }
 
@@ -401,7 +401,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
 
     for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) {
       bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg);
-      char* p = colDataGet(pColData, j);
+      char* p = colDataGetData(pColData, j);
 
       colDataAppend(pDstCol, j - startIndex, p, isNull);
     }
@@ -443,7 +443,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
       pStart += BitmapLen(pBlock->info.rows);
     }
 
-    uint32_t dataSize = colDataGetSize(pCol, numOfRows);
+    uint32_t dataSize = colDataGetLength(pCol, numOfRows);
 
     *(int32_t*) pStart = dataSize;
     pStart += sizeof(int32_t);
@@ -592,8 +592,8 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
       }
     }
 
-    void* left1  = colDataGet(pColInfoData, left);
-    void* right1 = colDataGet(pColInfoData, right);
+    void* left1  = colDataGetData(pColInfoData, left);
+    void* right1 = colDataGetData(pColInfoData, right);
 
     switch(pColInfoData->info.type) {
       case TSDB_DATA_TYPE_INT: {
@@ -632,7 +632,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co
         return code;
       }
     } else {
-      char* p = colDataGet(pSrc, tupleIndex);
+      char* p = colDataGetData(pSrc, tupleIndex);
       code = colDataAppend(pDst, numOfRows, p, false);
       if (code != TSDB_CODE_SUCCESS) {
         return code;
@@ -971,8 +971,8 @@ int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) {
 //      }
 //    }
 
-//    void* left1  = colDataGet(pColInfoData, left);
-//    void* right1 = colDataGet(pColInfoData, right);
+//    void* left1  = colDataGetData(pColInfoData, left);
+//    void* right1 = colDataGetData(pColInfoData, right);
 
 //    switch(pColInfoData->info.type) {
 //      case TSDB_DATA_TYPE_INT: {
@@ -1114,3 +1114,24 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
   tfree(pBlock);
   return NULL;
 }
+
+SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) {
+  int32_t numOfCols = pDataBlock->info.numOfCols;
+
+  SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock));
+  pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
+  pBlock->info.numOfCols = numOfCols;
+
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData  colInfo = {0};
+    SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
+    colInfo.info = p->info;
+    taosArrayPush(pBlock->pDataBlock, &colInfo);
+  }
+
+  return pBlock;
+}
+
+size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
+  return pageSize / (blockDataGetSerialRowSize(pBlock) + blockDataGetSerialMetaSize(pBlock));
+}
\ No newline at end of file

source/common/test/commonTests.cpp

@@ -162,7 +162,7 @@ TEST(testCase, Datablock_test) {
   ASSERT_EQ(colDataGetNumOfCols(b), 2);
   ASSERT_EQ(colDataGetNumOfRows(b), 40);
 
-  char* pData = colDataGet(p1, 3);
+  char* pData = colDataGetData(p1, 3);
   printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData));
 
   SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo));

source/libs/executor/inc/executil.h

@@ -69,8 +69,8 @@ typedef struct SResultRow {
 
 typedef struct SResultRowInfo {
   SResultRow** pResult;    // result list
-  int16_t      type:8;     // data type for hash key
-  int32_t      size:24;    // number of result set
+//  int16_t      type:8;     // data type for hash key
+  int32_t      size;       // number of result set
   int32_t      capacity;   // max capacity
   int32_t      curPos;     // current active result row index of pResult list
 } SResultRowInfo;
@@ -95,7 +95,7 @@ struct SUdfInfo;
 int32_t getOutputInterResultBufSize(struct STaskAttr* pQueryAttr);
 
 size_t  getResultRowSize(SArray* pExprInfo);
-int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t type);
+int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size);
 void    cleanupResultRowInfo(SResultRowInfo* pResultRowInfo);
 
 void    resetResultRowInfo(struct STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo);
@@ -105,7 +105,7 @@ void    closeAllResultRows(SResultRowInfo* pResultRowInfo);
 int32_t initResultRow(SResultRow *pResultRow);
 void    closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot);
 bool    isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot);
-void    clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type);
+void    clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow);
 
 struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset);
 

source/libs/executor/inc/executorimpl.h

@@ -445,16 +445,20 @@ typedef struct SOptrBasicInfo {
   int32_t         capacity;
 } SOptrBasicInfo;
 
-typedef struct SOptrBasicInfo STableIntervalOperatorInfo;
-
-typedef struct SAggOperatorInfo {
-  SOptrBasicInfo       binfo;
-  SDiskbasedBuf       *pResultBuf;           // query result buffer based on blocked-wised disk file
+typedef struct SAggSupporter {
   SHashObj*            pResultRowHashTable;  // quick locate the window object for each result
   SHashObj*            pResultRowListSet;    // used to check if current ResultRowInfo has ResultRow object or not
   SArray*              pResultRowArrayList;  // The array list that contains the Result rows
   char*                keyBuf;               // window key buffer
   SResultRowPool      *pool;  // The window result objects pool, all the resultRow Objects are allocated and managed by this object.
+} SAggSupporter;
+
+typedef struct SOptrBasicInfo STableIntervalOperatorInfo;
+
+typedef struct SAggOperatorInfo {
+  SOptrBasicInfo       binfo;
+  SDiskbasedBuf       *pResultBuf;           // query result buffer based on blocked-wised disk file
+  SAggSupporter        aggSup;
   STableQueryInfo     *current;
   uint32_t             groupId;
   SGroupResInfo        groupResInfo;
@@ -552,8 +556,6 @@ typedef struct SDistinctOperatorInfo {
 
 typedef struct SSortedMergeOperatorInfo {
   SOptrBasicInfo     binfo;
-
-//  SSDataBlock       *pDataBlock;
   bool               hasVarCol;
   
   SArray            *orderInfo;   // SArray<SBlockOrderInfo>
@@ -564,12 +566,16 @@ typedef struct SSortedMergeOperatorInfo {
 
   int32_t            bufPageSize;
   uint32_t           sortBufSize;  // max buffer size for in-memory sort
-  int32_t            numOfRowsInRes;
 
-  char**             prevRow;
   int32_t            resultRowFactor;
-  bool               multiGroupResults;
-  bool               hasGroupColData;
+  bool               hasGroupVal;
+
+  SDiskbasedBuf     *pTupleStore;  // keep the final results
+  int32_t            numOfResPerPage;
+
+  char**             groupVal;
+  SArray            *groupInfo;
+  SAggSupporter      aggSup;
 } SSortedMergeOperatorInfo;
 
 typedef struct SOrderOperatorInfo {
@@ -634,7 +640,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
 SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema,
                                       int32_t numOfOutput);
 SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
 
 // SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
 // SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);
@@ -682,9 +688,6 @@ int32_t checkForQueryBuf(size_t numOfTables);
 bool    checkNeedToCompressQueryCol(SQInfo* pQInfo);
 void    setQueryStatus(STaskRuntimeEnv* pRuntimeEnv, int8_t status);
 
-bool onlyQueryTags(STaskAttr* pQueryAttr);
-// void destroyUdfInfo(struct SUdfInfo* pUdfInfo);
-
 int32_t doDumpQueryResult(SQInfo* pQInfo, char* data, int8_t compressed, int32_t* compLen);
 
 size_t getResultSize(SQInfo* pQInfo, int64_t* numOfRows);

source/libs/executor/src/executil.c

@@ -53,8 +53,8 @@ int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) {
   return size;
 }
 
-int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) {
-  pResultRowInfo->type     = type;
+int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) {
+//  pResultRowInfo->type     = type;
   pResultRowInfo->size     = 0;
   pResultRowInfo->curPos  = -1;
   pResultRowInfo->capacity = size;
@@ -93,7 +93,7 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow
 
   for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
     SResultRow *pWindowRes = pResultRowInfo->pResult[i];
-    clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type);
+    clearResultRow(pRuntimeEnv, pWindowRes);
 
     int32_t groupIndex = 0;
     int64_t uid = 0;
@@ -136,7 +136,7 @@ void closeResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) {
   getResultRow(pResultRowInfo, slot)->closed = true;
 }
 
-void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_t type) {
+void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) {
   if (pResultRow == NULL) {
     return;
   }

source/libs/executor/src/tsort.c

@@ -123,23 +123,6 @@ int32_t sortAddSource(SSortHandle* pSortHandle, void* pSource) {
   taosArrayPush(pSortHandle->pOrderedSource, &pSource);
 }
 
-static SSDataBlock* createDataBlock(const SSDataBlock* pDataBlock) {
-  int32_t numOfCols = pDataBlock->info.numOfCols;
-
-  SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock));
-  pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
-  pBlock->info.numOfCols = numOfCols;
-
-  for(int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData  colInfo = {0};
-    SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    colInfo.info = p->info;
-    taosArrayPush(pBlock->pDataBlock, &colInfo);
-  }
-
-  return pBlock;
-}
-
 static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSources, SSDataBlock* pBlock, int32_t* sourceId) {
   SExternalMemSource* pSource = calloc(1, sizeof(SExternalMemSource));
   if (pSource == NULL) {
@@ -198,7 +181,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
 
   blockDataClearup(pDataBlock, pHandle->hasVarCol);
 
-  SSDataBlock* pBlock = createDataBlock(pDataBlock);
+  SSDataBlock* pBlock = createOneDataBlock(pDataBlock);
   int32_t code = doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
@@ -263,7 +246,7 @@ static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSou
     if (isNull) {
       colDataAppend(pColInfo, pBlock->info.rows, NULL, true);
     } else {
-      char* pData = colDataGet(pSrcColInfo, *rowIndex);
+      char* pData = colDataGetData(pSrcColInfo, *rowIndex);
       colDataAppend(pColInfo, pBlock->info.rows, pData, false);
     }
   }
@@ -279,15 +262,14 @@ static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwa
    */
   if (pSource->src.rowIndex >= pSource->src.pBlock->info.rows) {
     pSource->src.rowIndex = 0;
-    pSource->pageIndex += 1;
 
+    if (pHandle->type == SORT_SINGLESOURCE_SORT) {
       if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) {
         (*numOfCompleted) += 1;
         pSource->src.rowIndex = -1;
         pSource->pageIndex = -1;
         pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock);
       } else {
-      if (pHandle->type == SORT_SINGLESOURCE_SORT) {
         SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex);
 
         SFilePage* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo));
@@ -297,6 +279,7 @@ static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwa
         }
 
         releaseBufPage(pHandle->pBuf, pPage);
+      }
     } else {
       pSource->src.pBlock = pHandle->fetchfp(((SGenericSource*)pSource)->param);
       if (pSource->src.pBlock == NULL) {
@@ -305,7 +288,6 @@ static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwa
       }
     }
   }
-  }
 
   /*
    * Adjust loser tree otherwise, according to new candidate data
@@ -404,8 +386,8 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) {
       return pParam->nullFirst? -1:1;
     }
 
-    void* left1  = colDataGet(pLeftColInfoData, pLeftSource->src.rowIndex);
-    void* right1 = colDataGet(pRightColInfoData, pRightSource->src.rowIndex);
+    void* left1  = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex);
+    void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex);
 
     switch(pLeftColInfoData->info.type) {
       case TSDB_DATA_TYPE_INT: {
@@ -499,7 +481,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
       tMergeTreeDestroy(pHandle->pMergeTree);
       pHandle->numOfCompletedSources = 0;
 
-      SSDataBlock* pBlock = createDataBlock(pHandle->pDataBlock);
+      SSDataBlock* pBlock = createOneDataBlock(pHandle->pDataBlock);
       code = doAddNewExternalMemSource(pHandle->pBuf, pResList, pBlock, &pHandle->sourceId);
       if (code != 0) {
         return code;
@@ -545,7 +527,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) {
       }
 
       if (pHandle->pDataBlock == NULL) {
-        pHandle->pDataBlock = createDataBlock(pBlock);
+        pHandle->pDataBlock = createOneDataBlock(pBlock);
       }
 
       int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock);
@@ -646,6 +628,7 @@ STupleHandle* sortNextTuple(SSortHandle* pHandle) {
     return NULL;
   }
 
+  // All the data are hold in the buffer, no external sort is invoked.
   if (pHandle->inMemSort) {
     pHandle->tupleHandle.rowIndex += 1;
     if (pHandle->tupleHandle.rowIndex == pHandle->pDataBlock->info.rows) {
@@ -671,6 +654,7 @@ STupleHandle* sortNextTuple(SSortHandle* pHandle) {
     return NULL;
   }
 
+  // Get the adjusted value after the loser tree is updated.
   index = tMergeTreeGetChosenIndex(pHandle->pMergeTree);
   pSource = pHandle->cmpParam.pSources[index];
 
@@ -691,5 +675,5 @@ bool sortIsValueNull(STupleHandle* pVHandle, int32_t colIndex) {
 
 void* sortGetValue(STupleHandle* pVHandle, int32_t colIndex) {
   SColumnInfoData* pColInfo = TARRAY_GET_ELEM(pVHandle->pBlock->pDataBlock, colIndex);
-  return colDataGet(pColInfo, pVHandle->rowIndex);
+  return colDataGetData(pColInfo, pVHandle->rowIndex);
 }

source/libs/executor/test/executorTests.cpp

@@ -42,22 +42,21 @@ enum {
 };
 
 typedef struct SDummyInputInfo {
-  int32_t max;
+  int32_t      totalPages;     // numOfPages
   int32_t      current;
   int32_t      startVal;
   int32_t      type;
+  int32_t      numOfRowsPerPage;
   SSDataBlock* pBlock;
 } SDummyInputInfo;
 
 SSDataBlock* getDummyBlock(void* param, bool* newgroup) {
   SOperatorInfo* pOperator = static_cast<SOperatorInfo*>(param);
   SDummyInputInfo* pInfo = static_cast<SDummyInputInfo*>(pOperator->info);
-  if (pInfo->current >= pInfo->max) {
+  if (pInfo->current >= pInfo->totalPages) {
     return NULL;
   }
 
-  int32_t numOfRows = 1000;
-
   if (pInfo->pBlock == NULL) {
     pInfo->pBlock = static_cast<SSDataBlock*>(calloc(1, sizeof(SSDataBlock)));
 
@@ -67,8 +66,8 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) {
     colInfo.info.type = TSDB_DATA_TYPE_INT;
     colInfo.info.bytes = sizeof(int32_t);
     colInfo.info.colId = 1;
-    colInfo.pData = static_cast<char*>(calloc(numOfRows, sizeof(int32_t)));
-    colInfo.nullbitmap = static_cast<char*>(calloc(1, (numOfRows + 7) / 8));
+    colInfo.pData = static_cast<char*>(calloc(pInfo->numOfRowsPerPage, sizeof(int32_t)));
+    colInfo.nullbitmap = static_cast<char*>(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
 
     taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo);
 
@@ -91,7 +90,7 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) {
   char buf[128] = {0};
   char b1[128] = {0};
   int32_t v = 0;
-  for(int32_t i = 0; i < numOfRows; ++i) {
+  for(int32_t i = 0; i < pInfo->numOfRowsPerPage; ++i) {
     SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
 
     if (pInfo->type == data_desc) {
@@ -111,21 +110,22 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) {
 //    colDataAppend(pColInfo2, i, b1, false);
   }
 
-  pBlock->info.rows = numOfRows;
+  pBlock->info.rows = pInfo->numOfRowsPerPage;
   pBlock->info.numOfCols = 1;
 
   pInfo->current += 1;
   return pBlock;
 }
 
-SOperatorInfo* createDummyOperator(int32_t numOfBlocks, int32_t type) {
+SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_t rowsPerPage, int32_t type) {
   SOperatorInfo* pOperator = static_cast<SOperatorInfo*>(calloc(1, sizeof(SOperatorInfo)));
   pOperator->name = "dummyInputOpertor4Test";
   pOperator->exec = getDummyBlock;
 
   SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo));
-  pInfo->max = numOfBlocks;
-  pInfo->startVal = 1500000;
+  pInfo->totalPages = numOfBlocks;
+  pInfo->startVal   = startVal;
+  pInfo->numOfRowsPerPage = rowsPerPage;
   pInfo->type       = type;
 
   pOperator->info = pInfo;
@@ -257,7 +257,7 @@ TEST(testCase, inMem_sort_Test) {
   SColumnInfoData* pCol1 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 0));
   SColumnInfoData* pCol2 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 1));
   for(int32_t i = 0; i < pRes->info.rows; ++i) {
-    char* p = colDataGet(pCol2, i);
+    char* p = colDataGetData(pCol2, i);
     printf("%d: %d, %s\n", i, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p));
   }
 }
@@ -341,7 +341,7 @@ TEST(testCase, external_sort_Test) {
     SColumnInfoData* pCol1 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 0));
 //    SColumnInfoData* pCol2 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 1));
     for (int32_t i = 0; i < pRes->info.rows; ++i) {
-//      char* p = colDataGet(pCol2, i);
+//      char* p = colDataGetData(pCol2, i);
       printf("%d: %d\n", total++, ((int32_t*)pCol1->pData)[i]);
 //      printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p));
     }
@@ -357,6 +357,7 @@ TEST(testCase, external_sort_Test) {
   taosArrayDestroy(pOrderVal);
 }
 
+#endif
 
 TEST(testCase, sorted_merge_Test) {
   srand(time(NULL));
@@ -370,7 +371,12 @@ TEST(testCase, sorted_merge_Test) {
 
   SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo));
   SExprInfo *exp = static_cast<SExprInfo*>(calloc(1, sizeof(SExprInfo)));
-  exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res");
+  exp->base.resSchema = createSchema(TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 1, "count_result");
+  exp->base.pColumns = static_cast<SColumn*>(calloc(1, sizeof(SColumn)));
+  exp->base.pColumns->flag = TSDB_COL_NORMAL;
+  exp->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_INT, .bytes = 4};
+  exp->base.numOfCols = 1;
+
   taosArrayPush(pExprInfo, &exp);
 
   SExprInfo *exp1 = static_cast<SExprInfo*>(calloc(1, sizeof(SExprInfo)));
@@ -380,10 +386,10 @@ TEST(testCase, sorted_merge_Test) {
   int32_t numOfSources = 10;
   SOperatorInfo** plist = (SOperatorInfo**) calloc(numOfSources, sizeof(void*));
   for(int32_t i = 0; i < numOfSources; ++i) {
-    plist[i] = createDummyOperator(1, data_asc);
+    plist[i] = createDummyOperator(1, 1, 1, data_asc);
   }
 
-  SOperatorInfo* pOperator = createSortedMergeOperatorInfo(plist, numOfSources, pExprInfo, pOrderVal, NULL);
+  SOperatorInfo* pOperator = createSortedMergeOperatorInfo(plist, numOfSources, pExprInfo, pOrderVal, NULL, NULL);
 
   bool newgroup = false;
   SSDataBlock* pRes = NULL;
@@ -409,8 +415,8 @@ TEST(testCase, sorted_merge_Test) {
     SColumnInfoData* pCol1 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 0));
 //    SColumnInfoData* pCol2 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 1));
     for (int32_t i = 0; i < pRes->info.rows; ++i) {
-//      char* p = colDataGet(pCol2, i);
-      printf("%d: %d\n", total++, ((int32_t*)pCol1->pData)[i]);
+//      char* p = colDataGetData(pCol2, i);
+      printf("%d: %ld\n", total++, ((int64_t*)pCol1->pData)[i]);
 //      printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p));
     }
   }
@@ -424,5 +430,4 @@ TEST(testCase, sorted_merge_Test) {
   taosArrayDestroy(pExprInfo);
   taosArrayDestroy(pOrderVal);
 }
-#endif
 #pragma GCC diagnostic pop

source/libs/executor/test/executorUtilTests.cpp

@@ -123,8 +123,8 @@ int32_t docomp(const void* p1, const void* p2, void* param) {
       return pParam->nullFirst? -1:1;
     }
 
-    void* left1  = colDataGet(pLeftColInfoData, pLeftSource->src.rowIndex);
-    void* right1 = colDataGet(pRightColInfoData, pRightSource->src.rowIndex);
+    void* left1  = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex);
+    void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex);
 
     switch(pLeftColInfoData->info.type) {
       case TSDB_DATA_TYPE_INT: {
@@ -148,48 +148,15 @@ int32_t docomp(const void* p1, const void* p2, void* param) {
 }
 }  // namespace
 
-//TEST(testCase, inMem_sort_Test) {
-//  SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
-//  SOrder o = {.order = TSDB_ORDER_ASC};
-//  o.col.info.colId = 1;
-//  o.col.info.type = TSDB_DATA_TYPE_INT;
-//  taosArrayPush(pOrderVal, &o);
-//
-//  int32_t numOfRows = 1000;
-//  SBlockOrderInfo oi = {0};
-//  oi.order = TSDB_ORDER_ASC;
-//  oi.colIndex = 0;
-//  SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
-//  taosArrayPush(orderInfo, &oi);
-//
-//  SSortHandle* phandle = createSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, "test_abc");
-//  setFetchRawDataFp(phandle, getSingleColDummyBlock);
-//  sortAddSource(phandle, &numOfRows);
-//
-//  int32_t code = sortOpen(phandle);
-//  int32_t row = 1;
-//
-//  while(1) {
-//    STupleHandle* pTupleHandle = sortNextTuple(phandle);
-//    if (pTupleHandle == NULL) {
-//      break;
-//    }
-//
-//    void* v = sortGetValue(pTupleHandle, 0);
-//    printf("%d: %d\n", row++, *(int32_t*) v);
-//
-//  }
-//  destroySortHandle(phandle);
-//}
-//
-TEST(testCase, external_mem_sort_Test) {
+#if 0
+TEST(testCase, inMem_sort_Test) {
   SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
   SOrder o = {.order = TSDB_ORDER_ASC};
   o.col.info.colId = 1;
   o.col.info.type = TSDB_DATA_TYPE_INT;
   taosArrayPush(pOrderVal, &o);
 
-//  int32_t numOfRows = 1000;
+  int32_t numOfRows = 1000;
   SBlockOrderInfo oi = {0};
   oi.order = TSDB_ORDER_ASC;
   oi.colIndex = 0;
@@ -197,7 +164,40 @@ TEST(testCase, external_mem_sort_Test) {
   taosArrayPush(orderInfo, &oi);
 
   SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4, };
+  SSortHandle* phandle = createSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, &s, 1, "test_abc");
+  setFetchRawDataFp(phandle, getSingleColDummyBlock);
+  sortAddSource(phandle, &numOfRows);
 
+  int32_t code = sortOpen(phandle);
+  int32_t row = 1;
+
+  while(1) {
+    STupleHandle* pTupleHandle = sortNextTuple(phandle);
+    if (pTupleHandle == NULL) {
+      break;
+    }
+
+    void* v = sortGetValue(pTupleHandle, 0);
+    printf("%d: %d\n", row++, *(int32_t*) v);
+
+  }
+  destroySortHandle(phandle);
+}
+
+TEST(testCase, external_mem_sort_Test) {
+  SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
+  SOrder o = {.order = TSDB_ORDER_ASC};
+  o.col.info.colId = 1;
+  o.col.info.type = TSDB_DATA_TYPE_INT;
+  taosArrayPush(pOrderVal, &o);
+
+  SBlockOrderInfo oi = {0};
+  oi.order = TSDB_ORDER_ASC;
+  oi.colIndex = 0;
+  SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
+  taosArrayPush(orderInfo, &oi);
+
+  SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4, };
   SSortHandle* phandle = createSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, &s, 1, "test_abc");
   setFetchRawDataFp(phandle, getSingleColDummyBlock);
 
@@ -227,50 +227,52 @@ TEST(testCase, external_mem_sort_Test) {
   destroySortHandle(phandle);
 }
 
-//TEST(testCase, ordered_merge_sort_Test) {
-//  SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
-//  SOrder o = {.order = TSDB_ORDER_ASC};
-//  o.col.info.colId = 1;
-//  o.col.info.type = TSDB_DATA_TYPE_INT;
-//  taosArrayPush(pOrderVal, &o);
-//
-//  int32_t numOfRows = 1000;
-//  SBlockOrderInfo oi = {0};
-//  oi.order = TSDB_ORDER_ASC;
-//  oi.colIndex = 0;
-//  SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
-//  taosArrayPush(orderInfo, &oi);
-//
-//  SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4};
-//  SSortHandle* phandle = createSortHandle(orderInfo, false, SORT_MULTISOURCE_MERGE, 1024, 5, &s, 1,"test_abc");
-//  setFetchRawDataFp(phandle, getSingleColDummyBlock);
-//  setComparFn(phandle, docomp);
-//
-//  for(int32_t i = 0; i < 10; ++i) {
-//    SOperatorSource* p = static_cast<SOperatorSource*>(calloc(1, sizeof(SOperatorSource)));
-//    _info* c = static_cast<_info*>(calloc(1, sizeof(_info)));
-//    c->count    = 1;
-//    c->pageRows = 1000;
-//    c->startVal = 0;
-//
-//    p->param = c;
-//    sortAddSource(phandle, p);
-//  }
-//
-//  int32_t code = sortOpen(phandle);
-//  int32_t row = 1;
-//
-//  while(1) {
-//    STupleHandle* pTupleHandle = sortNextTuple(phandle);
-//    if (pTupleHandle == NULL) {
-//      break;
-//    }
-//
-//    void* v = sortGetValue(pTupleHandle, 0);
-//    printf("%d: %d\n", row++, *(int32_t*) v);
-//
-//  }
-//  destroySortHandle(phandle);
-//}
+TEST(testCase, ordered_merge_sort_Test) {
+  SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
+  SOrder o = {.order = TSDB_ORDER_ASC};
+  o.col.info.colId = 1;
+  o.col.info.type = TSDB_DATA_TYPE_INT;
+  taosArrayPush(pOrderVal, &o);
+
+  int32_t numOfRows = 1000;
+  SBlockOrderInfo oi = {0};
+  oi.order = TSDB_ORDER_ASC;
+  oi.colIndex = 0;
+  SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
+  taosArrayPush(orderInfo, &oi);
+
+  SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4};
+  SSortHandle* phandle = createSortHandle(orderInfo, false, SORT_MULTISOURCE_MERGE, 1024, 5, &s, 1,"test_abc");
+  setFetchRawDataFp(phandle, getSingleColDummyBlock);
+  setComparFn(phandle, docomp);
+
+  for(int32_t i = 0; i < 10; ++i) {
+    SGenericSource* p = static_cast<SGenericSource*>(calloc(1, sizeof(SGenericSource)));
+    _info* c = static_cast<_info*>(calloc(1, sizeof(_info)));
+    c->count    = 1;
+    c->pageRows = 1000;
+    c->startVal = 0;
+
+    p->param = c;
+    sortAddSource(phandle, p);
+  }
+
+  int32_t code = sortOpen(phandle);
+  int32_t row = 1;
+
+  while(1) {
+    STupleHandle* pTupleHandle = sortNextTuple(phandle);
+    if (pTupleHandle == NULL) {
+      break;
+    }
+
+    void* v = sortGetValue(pTupleHandle, 0);
+    printf("%d: %d\n", row++, *(int32_t*) v);
+
+  }
+  destroySortHandle(phandle);
+}
+
+#endif
 
 #pragma GCC diagnostic pop

source/libs/function/src/taggfunction.c

@@ -30,9 +30,10 @@
 #include "tcompression.h"
 //#include "queryLog.h"
 #include "tudf.h"
+#include "tep.h"
 
 #define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput))
-#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes)
+#define GET_INPUT_DATA(x, y) ((char*) colDataGetData((x)->pInput, (y)))
 
 #define GET_TS_LIST(x)    ((TSKEY*)((x)->ptsList))
 #define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)])
@@ -3817,7 +3818,7 @@ static void interp_function_impl(SqlFunctionCtx *pCtx) {
             skey = ekey;
           }
         }
-        assignVal(pCtx->pOutput, pCtx->pInput, pCtx->resDataInfo.bytes, pCtx->inputType);
+//        assignVal(pCtx->pOutput, pCtx->pInput, pCtx->resDataInfo.bytes, pCtx->inputType);
       } else if (type == TSDB_FILL_NEXT) {
         TSKEY ekey = skey;
         char* val = NULL;

source/libs/parser/src/queryInfoUtil.c

@@ -230,7 +230,7 @@ int32_t getExprFunctionId(SExprInfo *pExprInfo) {
 }
 
 void assignExprInfo(SExprInfo* dst, const SExprInfo* src) {
-  assert(dst != NULL && src != NULL);
+  assert(dst != NULL && src != NULL && src->base.numOfCols > 0);
 
   *dst = *src;
 #if 0