[td-11818] refactor sort, and sort operator.

include/common/tep.h

@@ -18,9 +18,6 @@ typedef struct SBlockOrderInfo {
   int32_t          order;
   int32_t          colIndex;
   SColumnInfoData *pColData;
-//  int32_t          type;
-//  int32_t          bytes;
-//  bool             hasNull;
 } SBlockOrderInfo;
 
 int  taosGetFqdnPortFromEp(const char *ep, SEp *pEp);
@@ -93,6 +90,8 @@ size_t blockDataGetRowSize(const SSDataBlock* pBlock);
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
 size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
 
+SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols);
+
 size_t blockDataNumOfRowsForSerialize(const SSDataBlock* pBlock, int32_t blockSize);
 
 int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);

include/util/tpagedbuf.h

@@ -157,7 +157,7 @@ void setBufPageDirty(SFilePage* pPageInfo, bool dirty);
  * Print the statistics when closing this buffer
  * @param pBuf
  */
-void printStatisBeforeClose(SDiskbasedBuf* pBuf);
+void setPrintStatis(SDiskbasedBuf* pBuf);
 
 /**
  * return buf statistics.

source/common/src/tep.c

@@ -411,7 +411,6 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
   return pDst;
 }
 
-
 /**
  *
  * +------------------+---------------+--------------------+
@@ -522,6 +521,22 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) {
   return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t);
 }
 
+SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols) {
+  SSchema* pSchema = calloc(pBlock->info.numOfCols, sizeof(SSchema));
+  for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+    pSchema[i].bytes = pColInfoData->info.bytes;
+    pSchema[i].type  = pColInfoData->info.type;
+    pSchema[i].colId = pColInfoData->info.colId;
+  }
+
+  if (numOfCols != NULL) {
+    *numOfCols = pBlock->info.numOfCols;
+  }
+
+  return pSchema;
+}
+
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
   ASSERT(pBlock != NULL);
   double rowSize = 0;

source/libs/executor/inc/executorimpl.h

@@ -15,6 +15,7 @@
 #ifndef TDENGINE_EXECUTORIMPL_H
 #define TDENGINE_EXECUTORIMPL_H
 
+#include "tsort.h"
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -557,10 +558,6 @@ typedef struct SSortMergeOperatorInfo {
   char**             currentGroupColData;
   SArray*            udfInfo;
   int32_t            numOfSources;
-//  char**             prevRow;
-//  int32_t            resultRowFactor;
-//  bool               multiGroupResults;
-//  bool               hasGroupColData;
 } SSortMergeOperatorInfo;
 
 typedef struct SMsortComparParam {
@@ -571,19 +568,16 @@ typedef struct SMsortComparParam {
 } SMsortComparParam;
 
 typedef struct SOrderOperatorInfo {
-  int32_t                 sourceId;
   uint32_t                sortBufSize;  // max buffer size for in-memory sort
   SSDataBlock            *pDataBlock;
   bool                    hasVarCol;    // has variable length column, such as binary/varchar/nchar
-  int32_t                 numOfCompleted;
-  SDiskbasedBuf          *pSortInternalBuf;
-  SMultiwayMergeTreeInfo *pMergeTree;
-  SArray                 *pSources;     // SArray<SExternalMemSource*>
+  SArray                 *orderInfo;
+  bool                    nullFirst;
+  SSortHandle            *pSortHandle;
+
   int32_t                 bufPageSize;
   int32_t                 numOfRowsInRes;
 
-  SMsortComparParam       cmpParam;
-
   // TODO extact struct
   int64_t                 startTs;       // sort start time
   uint64_t                sortElapsed;   // sort elapsed time, time to flush to disk not included.

source/libs/executor/inc/tsort.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TSORT_H
+#define TDENGINE_TSORT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "common.h"
+#include "os.h"
+
+enum {
+  SORT_MULTIWAY_MERGE = 0x1,
+  SORT_SINGLESOURCE = 0x2,
+};
+
+typedef struct SMultiMergeSource {
+  int32_t      type;
+  int32_t      rowIndex;
+  SSDataBlock *pBlock;
+} SMultiMergeSource;
+
+typedef struct SExternalMemSource {
+  SMultiMergeSource src;
+  SArray*           pageIdList;
+  int32_t           pageIndex;
+} SExternalMemSource;
+
+typedef struct SOperatorSource {
+  SMultiMergeSource src;
+  void* param;
+} SOperatorSource;
+
+typedef struct SSortHandle SSortHandle;
+typedef struct STupleHandle STupleHandle;
+
+typedef SSDataBlock* (*_sort_fetch_block_fn_t)(void* param);
+typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void* param);
+
+/**
+ *
+ * @param type
+ * @return
+ */
+SSortHandle* createSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr);
+
+/**
+ *
+ * @param pSortHandle
+ */
+void destroySortHandle(SSortHandle* pSortHandle);
+
+/**
+ *
+ * @param pHandle
+ * @return
+ */
+int32_t sortOpen(SSortHandle* pHandle);
+
+/**
+ *
+ * @param pHandle
+ * @return
+ */
+int32_t sortClose(SSortHandle* pHandle);
+
+/**
+ *
+ * @return
+ */
+int32_t setFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fp);
+
+/**
+ *
+ * @param pHandle
+ * @param fp
+ * @return
+ */
+int32_t setComparFn(SSortHandle* pHandle, _sort_merge_compar_fn_t fp);
+
+/**
+ *
+ * @param pHandle
+ * @param pSource
+ * @return success or failed
+ */
+int32_t sortAddSource(SSortHandle* pSortHandle, void* pSource);
+
+/**
+ *
+ * @param pHandle
+ * @return
+ */
+STupleHandle* sortNextTuple(SSortHandle* pHandle);
+
+/**
+ *
+ * @param pHandle
+ * @param colIndex
+ * @return
+ */
+bool sortIsValueNull(STupleHandle* pVHandle, int32_t colIndex);
+
+/**
+ *
+ * @param pHandle
+ * @param colIndex
+ * @return
+ */
+void* sortGetValue(STupleHandle* pVHandle, int32_t colIndex);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TSORT_H

source/libs/executor/test/executorTests.cpp

@@ -121,6 +121,7 @@ int main(int argc, char** argv) {
   return RUN_ALL_TESTS();
 }
 
+#if 0
 TEST(testCase, build_executor_tree_Test) {
   const char* msg = "{\n"
                     "\t\"Id\":\t{\n"
@@ -330,7 +331,7 @@ TEST(testCase, external_sort_Test) {
     }
   }
 
-  printStatisBeforeClose(((SOrderOperatorInfo*) pOperator->info)->pSortInternalBuf);
+//  setPrintStatis(((SOrderOperatorInfo*) pOperator->info)->pSortInternalBuf);
 
   int64_t s2 = taosGetTimestampUs();
   printf("total:%ld\n", s2 - s1);
@@ -341,4 +342,7 @@ TEST(testCase, external_sort_Test) {
   taosArrayDestroy(pExprInfo);
   taosArrayDestroy(pOrderVal);
 }
+
+#endif
+
 #pragma GCC diagnostic pop

source/libs/executor/test/executorUtilTests.cpp

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <executorimpl.h>
+#include <gtest/gtest.h>
+#include <tglobal.h>
+#include <tsort.h>
+#include <iostream>
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+#include "os.h"
+
+#include "executor.h"
+#include "stub.h"
+#include "taos.h"
+#include "tdef.h"
+#include "tep.h"
+#include "trpc.h"
+#include "tvariant.h"
+
+namespace {
+typedef struct {
+  int32_t startVal;
+  int32_t count;
+  int32_t pageRows;
+} _info;
+
+SSDataBlock* getSingleColDummyBlock(void* param) {
+  _info* pInfo = (_info*) param;
+  if (--pInfo->count < 0) {
+    return NULL;
+  }
+
+  SSDataBlock* pBlock = static_cast<SSDataBlock*>(calloc(1, sizeof(SSDataBlock)));
+  pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
+
+  SColumnInfoData colInfo = {0};
+  colInfo.info.type = TSDB_DATA_TYPE_INT;
+  colInfo.info.bytes = sizeof(int32_t);
+  colInfo.info.colId = 1;
+  colInfo.pData = static_cast<char*>(calloc(pInfo->pageRows, sizeof(int32_t)));
+  colInfo.nullbitmap = static_cast<char*>(calloc(1, (pInfo->pageRows + 7) / 8));
+
+  taosArrayPush(pBlock->pDataBlock, &colInfo);
+
+  for (int32_t i = 0; i < pInfo->pageRows; ++i) {
+    SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
+
+    int32_t v = ++pInfo->startVal;
+    colDataAppend(pColInfo, i, reinterpret_cast<const char*>(&v), false);
+  }
+
+  pBlock->info.rows = pInfo->pageRows;
+  pBlock->info.numOfCols = 1;
+  return pBlock;
+}
+
+int32_t docomp(const void* p1, const void* p2, void* param) {
+  int32_t pLeftIdx  = *(int32_t *)p1;
+  int32_t pRightIdx = *(int32_t *)p2;
+
+  SMsortComparParam *pParam = (SMsortComparParam *)param;
+  SOperatorSource** px = reinterpret_cast<SOperatorSource**>(pParam->pSources);
+
+  SArray *pInfo = pParam->orderInfo;
+
+  SOperatorSource* pLeftSource  = px[pLeftIdx];
+  SOperatorSource* pRightSource = px[pRightIdx];
+
+  // this input is exhausted, set the special value to denote this
+  if (pLeftSource->src.rowIndex == -1) {
+    return 1;
+  }
+
+  if (pRightSource->src.rowIndex == -1) {
+    return -1;
+  }
+
+  SSDataBlock* pLeftBlock = pLeftSource->src.pBlock;
+  SSDataBlock* pRightBlock = pRightSource->src.pBlock;
+
+  for(int32_t i = 0; i < pInfo->size; ++i) {
+    SBlockOrderInfo* pOrder = (SBlockOrderInfo*)TARRAY_GET_ELEM(pInfo, i);
+
+    SColumnInfoData* pLeftColInfoData = (SColumnInfoData*)TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->colIndex);
+
+    bool leftNull  = false;
+    if (pLeftColInfoData->hasNull) {
+      leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg);
+    }
+
+    SColumnInfoData* pRightColInfoData = (SColumnInfoData*) TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->colIndex);
+    bool rightNull = false;
+    if (pRightColInfoData->hasNull) {
+      rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg);
+    }
+
+    if (leftNull && rightNull) {
+      continue; // continue to next slot
+    }
+
+    if (rightNull) {
+      return pParam->nullFirst? 1:-1;
+    }
+
+    if (leftNull) {
+      return pParam->nullFirst? -1:1;
+    }
+
+    void* left1  = colDataGet(pLeftColInfoData, pLeftSource->src.rowIndex);
+    void* right1 = colDataGet(pRightColInfoData, pRightSource->src.rowIndex);
+
+    switch(pLeftColInfoData->info.type) {
+      case TSDB_DATA_TYPE_INT: {
+        int32_t leftv = *(int32_t*)left1;
+        int32_t rightv = *(int32_t*)right1;
+
+        if (leftv == rightv) {
+          break;
+        } else {
+          if (pOrder->order == TSDB_ORDER_ASC) {
+            return leftv < rightv? -1 : 1;
+          } else {
+            return leftv < rightv? 1 : -1;
+          }
+        }
+      }
+      default:
+        assert(0);
+    }
+  }
+}
+}  // namespace
+
+//TEST(testCase, inMem_sort_Test) {
+//  SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
+//  SOrder o = {.order = TSDB_ORDER_ASC};
+//  o.col.info.colId = 1;
+//  o.col.info.type = TSDB_DATA_TYPE_INT;
+//  taosArrayPush(pOrderVal, &o);
+//
+//  int32_t numOfRows = 1000;
+//  SBlockOrderInfo oi = {0};
+//  oi.order = TSDB_ORDER_ASC;
+//  oi.colIndex = 0;
+//  SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
+//  taosArrayPush(orderInfo, &oi);
+//
+//  SSortHandle* phandle = createSortHandle(orderInfo, false, SORT_SINGLESOURCE, 1024, 5, "test_abc");
+//  setFetchRawDataFp(phandle, getSingleColDummyBlock);
+//  sortAddSource(phandle, &numOfRows);
+//
+//  int32_t code = sortOpen(phandle);
+//  int32_t row = 1;
+//
+//  while(1) {
+//    STupleHandle* pTupleHandle = sortNextTuple(phandle);
+//    if (pTupleHandle == NULL) {
+//      break;
+//    }
+//
+//    void* v = sortGetValue(pTupleHandle, 0);
+//    printf("%d: %d\n", row++, *(int32_t*) v);
+//
+//  }
+//  destroySortHandle(phandle);
+//}
+//
+//TEST(testCase, external_mem_sort_Test) {
+//  totalcount = 50;
+//  startVal = 100000;
+//
+//  SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
+//  SOrder o = {.order = TSDB_ORDER_ASC};
+//  o.col.info.colId = 1;
+//  o.col.info.type = TSDB_DATA_TYPE_INT;
+//  taosArrayPush(pOrderVal, &o);
+//
+//  int32_t numOfRows = 1000;
+//  SBlockOrderInfo oi = {0};
+//  oi.order = TSDB_ORDER_ASC;
+//  oi.colIndex = 0;
+//  SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
+//  taosArrayPush(orderInfo, &oi);
+//
+//  SSortHandle* phandle = createSortHandle(orderInfo, false, SORT_SINGLESOURCE, 1024, 5, "test_abc");
+//  setFetchRawDataFp(phandle, getSingleColDummyBlock);
+//  sortAddSource(phandle, &numOfRows);
+//
+//  int32_t code = sortOpen(phandle);
+//  int32_t row = 1;
+//
+//  while(1) {
+//    STupleHandle* pTupleHandle = sortNextTuple(phandle);
+//    if (pTupleHandle == NULL) {
+//      break;
+//    }
+//
+//    void* v = sortGetValue(pTupleHandle, 0);
+//    printf("%d: %d\n", row++, *(int32_t*) v);
+//
+//  }
+//  destroySortHandle(phandle);
+//}
+
+TEST(testCase, ordered_merge_sort_Test) {
+  SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
+  SOrder o = {.order = TSDB_ORDER_ASC};
+  o.col.info.colId = 1;
+  o.col.info.type = TSDB_DATA_TYPE_INT;
+  taosArrayPush(pOrderVal, &o);
+
+  int32_t numOfRows = 1000;
+  SBlockOrderInfo oi = {0};
+  oi.order = TSDB_ORDER_ASC;
+  oi.colIndex = 0;
+  SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
+  taosArrayPush(orderInfo, &oi);
+
+  SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4};
+  SSortHandle* phandle = createSortHandle(orderInfo, false, SORT_MULTIWAY_MERGE, 1024, 5, &s, 1,"test_abc");
+  setFetchRawDataFp(phandle, getSingleColDummyBlock);
+  setComparFn(phandle, docomp);
+
+
+  for(int32_t i = 0; i < 10; ++i) {
+    SOperatorSource* p = static_cast<SOperatorSource*>(calloc(1, sizeof(SOperatorSource)));
+    _info* c = static_cast<_info*>(calloc(1, sizeof(_info)));
+    c->count    = 1;
+    c->pageRows = 1000;
+    c->startVal = 0;
+
+    p->param = c;
+    sortAddSource(phandle, p);
+  }
+
+  int32_t code = sortOpen(phandle);
+  int32_t row = 1;
+
+  while(1) {
+    STupleHandle* pTupleHandle = sortNextTuple(phandle);
+    if (pTupleHandle == NULL) {
+      break;
+    }
+
+    void* v = sortGetValue(pTupleHandle, 0);
+    printf("%d: %d\n", row++, *(int32_t*) v);
+
+  }
+  destroySortHandle(phandle);
+}
+
+#pragma GCC diagnostic pop

source/util/src/tpagedbuf.c

@@ -53,7 +53,7 @@ typedef struct SDiskbasedBuf {
 
 static void printStatisData(const SDiskbasedBuf* pBuf);
 
-  int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) {
+int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) {
   *pBuf = calloc(1, sizeof(SDiskbasedBuf));
 
   SDiskbasedBuf* pResBuf = *pBuf;
@@ -569,7 +569,7 @@ void setBufPageDirty(SFilePage* pPage, bool dirty) {
   ppi->dirty = dirty;
 }
 
-void printStatisBeforeClose(SDiskbasedBuf* pBuf) {
+void setPrintStatis(SDiskbasedBuf* pBuf) {
   pBuf->printStatis = true;
 }