executorimpl.c 316.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15

16
#include "os.h"
17 18 19

#include "tep.h"
#include "tsort.h"
S
Shengliang Guan 已提交
20
#include "texception.h"
H
Haojun Liao 已提交
21
#include "parser.h"
22
#include "tglobal.h"
H
Haojun Liao 已提交
23
#include "tmsg.h"
24
#include "ttime.h"
H
Haojun Liao 已提交
25

26 27 28 29
#include "executorimpl.h"
#include "function.h"
#include "tcompare.h"
#include "tcompression.h"
H
Haojun Liao 已提交
30
#include "thash.h"
31
#include "ttypes.h"
H
Haojun Liao 已提交
32
#include "query.h"
L
Liu Jicong 已提交
33
#include "vnode.h"
34
#include "tsdb.h"
35

H
Haojun Liao 已提交
36
#define IS_MAIN_SCAN(runtime)          ((runtime)->scanFlag == MAIN_SCAN)
37 38
#define IS_REVERSE_SCAN(runtime)       ((runtime)->scanFlag == REVERSE_SCAN)
#define IS_REPEAT_SCAN(runtime)        ((runtime)->scanFlag == REPEAT_SCAN)
H
Haojun Liao 已提交
39
#define SET_MAIN_SCAN_FLAG(runtime)  ((runtime)->scanFlag = MAIN_SCAN)
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN)

#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey))
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))

#define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0}

#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)

#define MULTI_KEY_DELIM  "-"

enum {
  TS_JOIN_TS_EQUAL       = 0,
  TS_JOIN_TS_NOT_EQUALS  = 1,
  TS_JOIN_TAG_NOT_EQUALS = 2,
};

typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP   = 2,
} SResultTsInterpType;

#if 0
static UNUSED_FUNC void *u_malloc (size_t __size) {
  uint32_t v = rand();

  if (v % 1000 <= 0) {
    return NULL;
  } else {
    return malloc(__size);
  }
}

static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) {
  uint32_t v = rand();
  if (v % 1000 <= 0) {
    return NULL;
  } else {
    return calloc(num, __size);
  }
}

static UNUSED_FUNC void* u_realloc(void* p, size_t __size) {
  uint32_t v = rand();
  if (v % 5 <= 1) {
    return NULL;
  } else {
    return realloc(p, __size);
  }
}

#define calloc  u_calloc
#define malloc  u_malloc
#define realloc u_realloc
#endif

#define CLEAR_QUERY_STATUS(q, st)   ((q)->status &= (~(st)))
#define GET_NUM_OF_TABLEGROUP(q)    taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList)
#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0)

#define TSKEY_MAX_ADD(a,b)                 \
do {                                       \
  if (a < 0) { a = a + b; break;}          \
  if (sizeof(a) == sizeof(int32_t)) {      \
   if((b) > 0 && ((b) >= INT32_MAX - (a))){\
     a = INT32_MAX;                        \
   } else {                                \
     a = a + b;                            \
   }                                       \
  } else {                                 \
   if((b) > 0 && ((b) >= INT64_MAX - (a))){\
     a = INT64_MAX;                        \
   } else {                                \
     a = a + b;                            \
   }                                       \
  }                                        \
} while(0)                                 

#define TSKEY_MIN_SUB(a,b)                 \
do {                                       \
  if (a >= 0) { a = a + b; break;}         \
  if (sizeof(a) == sizeof(int32_t)){       \
   if((b) < 0 && ((b) <= INT32_MIN - (a))){\
     a = INT32_MIN;                        \
   } else {                                \
     a = a + b;                            \
   }                                       \
  } else {                                 \
    if((b) < 0 && ((b) <= INT64_MIN-(a))) {\
     a = INT64_MIN;                        \
    } else {                               \
     a = a + b;                            \
    }                                      \
  }                                        \
} while (0)

int32_t getMaximumIdleDurationSec() {
S
Shengliang Guan 已提交
137
  return tsShellActivityTimer * 2;
138 139 140 141
}

static int32_t getExprFunctionId(SExprInfo *pExprInfo) {
  assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_UNARYEXPR_NODE);
142
  return 0;
143 144
}

H
Haojun Liao 已提交
145 146 147 148 149
static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
150 151 152
    return;
  }

H
Haojun Liao 已提交
153
  int64_t key = tw->skey, interval = pInterval->interval;
154
  //convert key to second
H
Haojun Liao 已提交
155
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;
156

H
Haojun Liao 已提交
157
  if (pInterval->intervalUnit == 'y') {
158 159 160 161 162 163 164 165 166 167
    interval *= 12;
  }

  struct tm tm;
  time_t t = (time_t)key;
  localtime_r(&t, &tm);

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
H
Haojun Liao 已提交
168
  tw->skey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision);
169 170 171 172

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
H
Haojun Liao 已提交
173
  tw->ekey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision);
174 175 176 177 178

  tw->ekey -= 1;
}

static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes);
H
Haojun Liao 已提交
179
static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SqlFunctionCtx* pCtx,
180 181
                               int32_t numOfCols, int32_t* rowCellInfoOffset);

H
Haojun Liao 已提交
182
void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset);
H
Haojun Liao 已提交
183
static bool functionNeedToExecute(SqlFunctionCtx *pCtx);
184

H
Haojun Liao 已提交
185
static void setBlockStatisInfo(SqlFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn);
186 187

static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo);
H
Haojun Liao 已提交
188
static bool hasMainOutput(STaskAttr *pQueryAttr);
189 190 191

static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols);

H
Haojun Liao 已提交
192
static int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo);
193 194
static void releaseQueryBuf(size_t numOfTables);
static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order);
H
Haojun Liao 已提交
195
//static STsdbQueryCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win);
196 197 198 199
static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo);

static void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream);

H
Haojun Liao 已提交
200
static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr);
201 202 203 204 205 206 207 208 209 210 211 212 213 214

static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
static void destroySFillOperatorInfo(void* param, int32_t numOfOutput);
static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput);
static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput);
static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput);
static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput);
static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput);
static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput);
static void destroyAggOperatorInfo(void* param, int32_t numOfOutput);
static void destroyOperatorInfo(SOperatorInfo* pOperator);

static void doSetOperatorCompleted(SOperatorInfo* pOperator) {
  pOperator->status = OP_EXEC_DONE;
H
Haojun Liao 已提交
215
  if (pOperator->pTaskInfo != NULL) {
216
    setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
217 218 219
  }
}

H
Haojun Liao 已提交
220
static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity);
221 222

static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock);
H
Haojun Liao 已提交
223
static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex);
224

H
Haojun Liao 已提交
225
static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
H
Haojun Liao 已提交
226 227
static void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win);

H
Haojun Liao 已提交
228
static void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo);
H
Haojun Liao 已提交
229 230 231
static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable);
static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr);
static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes);
232
static void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId, SExecTaskInfo* pTaskInfo);
233

H
Haojun Liao 已提交
234
SArray* getOrderCheckColumns(STaskAttr* pQuery);
235 236

typedef struct SRowCompSupporter {
H
Haojun Liao 已提交
237
  STaskRuntimeEnv *pRuntimeEnv;
238 239 240 241 242 243 244 245 246
  int16_t           dataOffset;
  __compar_fn_t     comFunc;
} SRowCompSupporter;

static int compareRowData(const void *a, const void *b, const void *userData) {
  const SResultRow *pRow1 = (const SResultRow *)a;
  const SResultRow *pRow2 = (const SResultRow *)b;

  SRowCompSupporter *supporter  = (SRowCompSupporter *)userData;
H
Haojun Liao 已提交
247
  STaskRuntimeEnv* pRuntimeEnv =  supporter->pRuntimeEnv;
248

249 250
  SFilePage *page1 = getBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId);
  SFilePage *page2 = getBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId);
251 252 253 254 255 256 257 258

  int16_t offset = supporter->dataOffset;
  char *in1  = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset);
  char *in2  = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset);

  return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0;
}

H
Haojun Liao 已提交
259
static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, STaskRuntimeEnv *pRuntimeEnv, SSDataBlock* pDataBlock) {
260 261 262 263 264 265 266 267
  SArray *columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr);
  size_t size = taosArrayGetSize(columnOrderList);
  taosArrayDestroy(columnOrderList);

  if (size <= 0) {
    return;
  }

H
Haojun Liao 已提交
268
  int32_t orderId = pRuntimeEnv->pQueryAttr->order.col.info.colId;
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308
  if (orderId <= 0) {
    return;
  }

  bool found = false;
  int16_t dataOffset = 0;

  for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) {
    SColumnInfoData* pColInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock->pDataBlock, j);
    if (orderId == j) {
      found = true;
      break;
    }

    dataOffset += pColInfoData->info.bytes;
  }

  if (found == false) {
    return;
  }

  int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resSchema.type;

  SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)};
  taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support);
}

//setup the output buffer for each operator
SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) {
  const static int32_t minSize = 8;

  SSDataBlock *res = calloc(1, sizeof(SSDataBlock));
  res->info.numOfCols = numOfOutput;
  res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData));
  for (int32_t i = 0; i < numOfOutput; ++i) {
    SColumnInfoData idata = {{0}};
    idata.info.type  = pExpr[i].base.resSchema.type;
    idata.info.bytes = pExpr[i].base.resSchema.bytes;
    idata.info.colId = pExpr[i].base.resSchema.colId;

dengyihao's avatar
dengyihao 已提交
309
    int32_t size = TMAX(idata.info.bytes * numOfRows, minSize);
310 311 312 313 314 315 316
    idata.pData = calloc(1, size);  // at least to hold a pointer on x64 platform
    taosArrayPush(res->pDataBlock, &idata);
  }

  return res;
}

H
Haojun Liao 已提交
317 318 319 320 321 322 323 324 325
SSDataBlock* createOutputBuf_rv(SArray* pExprInfo, int32_t numOfRows) {
  size_t numOfOutput = taosArrayGetSize(pExprInfo);

  SSDataBlock *res = calloc(1, sizeof(SSDataBlock));
  res->info.numOfCols = numOfOutput;
  res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData));

  for (int32_t i = 0; i < numOfOutput; ++i) {
    SColumnInfoData idata = {{0}};
H
Haojun Liao 已提交
326
    SExprInfo* pExpr = taosArrayGetP(pExprInfo, i);
H
Haojun Liao 已提交
327 328 329 330 331 332 333

    idata.info.type  = pExpr->base.resSchema.type;
    idata.info.bytes = pExpr->base.resSchema.bytes;
    idata.info.colId = pExpr->base.resSchema.colId;
    taosArrayPush(res->pDataBlock, &idata);
  }

H
Haojun Liao 已提交
334
  blockDataEnsureCapacity(res, numOfRows);
H
Haojun Liao 已提交
335 336 337
  return res;
}

H
Haojun Liao 已提交
338 339 340 341 342 343 344 345 346 347 348 349 350 351 352
SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) {
  int32_t numOfCols = LIST_LENGTH(pNode->pSlots);
  SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock));
  pBlock->info.numOfCols = numOfCols;
  pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));

  for(int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData idata = {{0}};
    SSlotDescNode* pDescNode = nodesListGetNode(pNode->pSlots, i);
    idata.info.type  = pDescNode->dataType.type;
    idata.info.bytes = pDescNode->dataType.bytes;
    idata.info.slotId = pDescNode->slotId;
  }
}

H
Haojun Liao 已提交
353
static bool isSelectivityWithTagsQuery(SqlFunctionCtx *pCtx, int32_t numOfOutput) {
354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
  return true;
//  bool    hasTags = false;
//  int32_t numOfSelectivity = 0;
//
//  for (int32_t i = 0; i < numOfOutput; ++i) {
//    int32_t functId = pCtx[i].functionId;
//    if (functId == FUNCTION_TAG_DUMMY || functId == FUNCTION_TS_DUMMY) {
//      hasTags = true;
//      continue;
//    }
//
//    if ((aAggs[functId].status & FUNCSTATE_SELECTIVITY) != 0) {
//      numOfSelectivity++;
//    }
//  }
//
//  return (numOfSelectivity > 0 && hasTags);
}

H
Haojun Liao 已提交
373
static bool isProjQuery(STaskAttr *pQueryAttr) {
374 375 376 377 378 379 380 381 382 383
  for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
    int32_t functId = getExprFunctionId(&pQueryAttr->pExpr1[i]);
    if (functId != FUNCTION_PRJ && functId != FUNCTION_TAGPRJ) {
      return false;
    }
  }

  return true;
}

H
Haojun Liao 已提交
384 385
static bool hasNull(SColumn* pColumn, SColumnDataAgg *pStatis) {
  if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) || pColumn->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
386 387 388 389 390 391 392 393 394 395
    return false;
  }

  if (pStatis != NULL && pStatis->numOfNull == 0) {
    return false;
  }

  return true;
}

H
Haojun Liao 已提交
396
static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) {
397 398
  int64_t newCapacity = 0;

399 400 401 402 403 404 405 406 407 408 409
  // more than the capacity, reallocate the resources
  if (pResultRowInfo->size < pResultRowInfo->capacity) {
    return;
  }

  if (pResultRowInfo->capacity > 10000) {
    newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25);
  } else {
    newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5);
  }

H
Haojun Liao 已提交
410 411 412 413
  if (newCapacity == pResultRowInfo->capacity) {
    newCapacity += 4;
  }

414 415
  char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES));
  if (t == NULL) {
H
Haojun Liao 已提交
416
    longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY);
417 418 419 420 421 422 423 424 425 426
  }

  pResultRowInfo->pResult = (SResultRow **)t;

  int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity;
  memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc);

  pResultRowInfo->capacity = (int32_t)newCapacity;
}

H
Haojun Liao 已提交
427
static bool chkResultRowFromKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData,
428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
                                             int16_t bytes, bool masterscan, uint64_t uid) {
  bool existed = false;
  SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid);

  SResultRow **p1 =
      (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));

  // in case of repeat scan/reverse scan, no new time window added.
  if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQueryAttr)) {
    if (!masterscan) {  // the *p1 may be NULL in case of sliding+offset exists.
      return p1 != NULL;
    }

    if (p1 != NULL) {
      if (pResultRowInfo->size == 0) {
        existed = false;
        assert(pResultRowInfo->curPos == -1);
      } else if (pResultRowInfo->size == 1) {
        existed = (pResultRowInfo->pResult[0] == (*p1));
      } else {  // check if current pResultRowInfo contains the existed pResultRow
        SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid, pResultRowInfo);
        int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes));
        if (index != NULL) {
          existed = true;
        } else {
          existed = false;
        }
      }
    }

    return existed;
  }

  return p1 != NULL;
}


H
Haojun Liao 已提交
465
static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid,
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
                                          char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId) {
  bool existed = false;
  SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tableGroupId);

  SResultRow **p1 =
      (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));

  // in case of repeat scan/reverse scan, no new time window added.
  if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQueryAttr)) {
    if (!masterscan) {  // the *p1 may be NULL in case of sliding+offset exists.
      return (p1 != NULL)? *p1:NULL;
    }

    if (p1 != NULL) {
      if (pResultRowInfo->size == 0) {
        existed = false;
        assert(pResultRowInfo->curPos == -1);
      } else if (pResultRowInfo->size == 1) {
        existed = (pResultRowInfo->pResult[0] == (*p1));
        pResultRowInfo->curPos = 0;
      } else {  // check if current pResultRowInfo contains the existed pResultRow
        SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo);
        int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes));
        if (index != NULL) {
          pResultRowInfo->curPos = (int32_t) *index;
          existed = true;
        } else {
          existed = false;
        }
      }
    }
  } else {
    // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object.
    if (p1 != NULL) {
      return *p1;
    }
  }

  if (!existed) {
H
Haojun Liao 已提交
505
//    prepareResultListBuffer(pResultRowInfo, pRuntimeEnv);
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538

    SResultRow *pResult = NULL;
    if (p1 == NULL) {
      pResult = getNewResultRow(pRuntimeEnv->pool);
      int32_t ret = initResultRow(pResult);
      if (ret != TSDB_CODE_SUCCESS) {
        longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }

      // add a new result set for a new group
      taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES);
      SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult};
      taosArrayPush(pRuntimeEnv->pResultRowArrayList, &cell);
    } else {
      pResult = *p1;
    }

    pResultRowInfo->curPos = pResultRowInfo->size;
    pResultRowInfo->pResult[pResultRowInfo->size++] = pResult;

    int64_t index = pResultRowInfo->curPos;
    SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo);
    taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES);
  }

  // too many time window in query
  if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) {
    longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
  }

  return pResultRowInfo->pResult[pResultRowInfo->curPos];
}

H
Haojun Liao 已提交
539
static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes,
540
    bool masterscan, uint64_t tableGroupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) {
H
Haojun Liao 已提交
541
  bool existed = false;
542
  SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, tableGroupId);
H
Haojun Liao 已提交
543 544

  SResultRow **p1 =
545
      (SResultRow **)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
H
Haojun Liao 已提交
546 547 548 549 550 551 552 553 554 555 556 557 558 559 560

  // in case of repeat scan/reverse scan, no new time window added.
  if (isIntervalQuery) {
    if (!masterscan) {  // the *p1 may be NULL in case of sliding+offset exists.
      return (p1 != NULL)? *p1:NULL;
    }

    if (p1 != NULL) {
      if (pResultRowInfo->size == 0) {
        existed = false;
        assert(pResultRowInfo->curPos == -1);
      } else if (pResultRowInfo->size == 1) {
        existed = (pResultRowInfo->pResult[0] == (*p1));
        pResultRowInfo->curPos = 0;
      } else {  // check if current pResultRowInfo contains the existed pResultRow
561 562
        SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo);
        int64_t* index = taosHashGet(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes));
H
Haojun Liao 已提交
563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
        if (index != NULL) {
          pResultRowInfo->curPos = (int32_t) *index;
          existed = true;
        } else {
          existed = false;
        }
      }
    }
  } else {
    // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object.
    if (p1 != NULL) {
      return *p1;
    }
  }

  if (!existed) {
    prepareResultListBuffer(pResultRowInfo, pTaskInfo->env);

    SResultRow *pResult = NULL;
    if (p1 == NULL) {
583
      pResult = getNewResultRow(pSup->pool);
H
Haojun Liao 已提交
584 585 586 587 588 589
      int32_t ret = initResultRow(pResult);
      if (ret != TSDB_CODE_SUCCESS) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }

      // add a new result set for a new group
590
      taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES);
H
Haojun Liao 已提交
591
      SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult};
592
      taosArrayPush(pSup->pResultRowArrayList, &cell);
H
Haojun Liao 已提交
593 594 595 596 597 598 599 600
    } else {
      pResult = *p1;
    }

    pResultRowInfo->curPos = pResultRowInfo->size;
    pResultRowInfo->pResult[pResultRowInfo->size++] = pResult;

    int64_t index = pResultRowInfo->curPos;
601 602
    SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo);
    taosHashPut(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES);
H
Haojun Liao 已提交
603 604 605 606 607 608 609 610 611 612
  }

  // too many time window in query
  if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
  }

  return pResultRowInfo->pResult[pResultRowInfo->curPos];
}

H
Haojun Liao 已提交
613 614 615
static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, TSKEY ekey, bool ascQuery) {
  if (ascQuery) {
    getAlignQueryTimeWindow(pInterval, precision, ts, ts, ekey, w);
616 617
  } else {
    // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
H
Haojun Liao 已提交
618
    getAlignQueryTimeWindow(pInterval, precision, ts, ekey, ts, w);
619 620 621

    int64_t key = w->skey;
    while(key < ts) { // moving towards end
H
Haojun Liao 已提交
622 623
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision);
624
      } else {
H
Haojun Liao 已提交
625
        key += pInterval->sliding;
626 627 628 629 630 631 632 633 634 635 636 637
      }

      if (key >= ts) {
        break;
      }

      w->skey = key;
    }
  }
}

// get the correct time window according to the handled timestamp
H
Haojun Liao 已提交
638
static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t precision, STimeWindow* win) {
639 640 641
  STimeWindow w = {0};

 if (pResultRowInfo->curPos == -1) {  // the first window, from the previous stored value
H
Haojun Liao 已提交
642
    getInitialStartTimeWindow(pInterval, precision, ts, &w, win->ekey, true);
643

H
Haojun Liao 已提交
644 645
    if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
      w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
646
    } else {
H
Haojun Liao 已提交
647
      w.ekey = w.skey + pInterval->interval - 1;
648 649 650 651 652 653
    }
  } else {
    w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win;
  }

  if (w.skey > ts || w.ekey < ts) {
H
Haojun Liao 已提交
654 655 656
    if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
      w.skey = taosTimeTruncate(ts, pInterval, precision);
      w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
657 658 659 660
    } else {
      int64_t st = w.skey;

      if (st > ts) {
H
Haojun Liao 已提交
661
        st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
662 663
      }

H
Haojun Liao 已提交
664
      int64_t et = st + pInterval->interval - 1;
665
      if (et < ts) {
H
Haojun Liao 已提交
666
        st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
667 668 669
      }

      w.skey = st;
H
Haojun Liao 已提交
670
      w.ekey = w.skey + pInterval->interval - 1;
671 672 673 674 675 676 677
    }
  }

  /*
   * query border check, skey should not be bounded by the query time range, since the value skey will
   * be used as the time window index value. So we only change ekey of time window accordingly.
   */
H
Haojun Liao 已提交
678
//  ASSERT(win->skey <= win->ekey); // todo no need this
679 680 681 682
  return w;
}

// get the correct time window according to the handled timestamp
H
Haojun Liao 已提交
683
static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, STaskAttr *pQueryAttr) {
684 685 686
  STimeWindow w = {0};

 if (pResultRowInfo->curPos == -1) {  // the first window, from the previous stored value
H
Haojun Liao 已提交
687
//    getInitialStartTimeWindow(pQueryAttr, ts, &w);
688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709

    if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') {
      w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1;
    } else {
      w.ekey = w.skey + pQueryAttr->interval.interval - 1;
    }
  } else {
    w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win;
  }

  /*
   * query border check, skey should not be bounded by the query time range, since the value skey will
   * be used as the time window index value. So we only change ekey of time window accordingly.
   */
  if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) {
    w.ekey = pQueryAttr->window.ekey;
  }

  return w;
}

// a new buffer page for each table. Needs to opt this design
710
static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedBuf *pResultBuf, int32_t tid, uint32_t size) {
711 712 713 714 715 716 717 718 719 720 721
  if (pWindowRes->pageId != -1) {
    return 0;
  }

  SFilePage *pData = NULL;

  // in the first scan, new space needed for results
  int32_t pageId = -1;
  SIDList list = getDataBufPagesIdList(pResultBuf, tid);

  if (taosArrayGetSize(list) == 0) {
H
Haojun Liao 已提交
722
    pData = getNewBufPage(pResultBuf, tid, &pageId);
723 724
  } else {
    SPageInfo* pi = getLastPageInfo(list);
725
    pData = getBufPage(pResultBuf, getPageId(pi));
726
    pageId = getPageId(pi);
727

728
    if (pData->num + size > getBufPageSize(pResultBuf)) {
729
      // release current page first, and prepare the next one
730
      releaseBufPageInfo(pResultBuf, pi);
H
Haojun Liao 已提交
731
      pData = getNewBufPage(pResultBuf, tid, &pageId);
732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753
      if (pData != NULL) {
        assert(pData->num == 0);  // number of elements must be 0 for new allocated buffer
      }
    }
  }

  if (pData == NULL) {
    return -1;
  }

  // set the number of rows in current disk page
  if (pWindowRes->pageId == -1) {  // not allocated yet, allocate new buffer
    pWindowRes->pageId = pageId;
    pWindowRes->offset = (int32_t)pData->num;

    pData->num += size;
    assert(pWindowRes->pageId >= 0);
  }

  return 0;
}

H
Haojun Liao 已提交
754
static bool chkWindowOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win,
H
Haojun Liao 已提交
755
                                       bool masterscan, SResultRow **pResult, int64_t groupId, SqlFunctionCtx* pCtx,
756 757 758 759 760
                                       int32_t numOfOutput, int32_t* rowCellInfoOffset) {
  assert(win->skey <= win->ekey);
  return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId);
}

H
Haojun Liao 已提交
761
static int32_t setResultOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win,
H
Haojun Liao 已提交
762
                                       bool masterscan, SResultRow **pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
763 764
                                       int32_t numOfOutput, int32_t* rowCellInfoOffset) {
  assert(win->skey <= win->ekey);
765
  SDiskbasedBuf *pResultBuf = pRuntimeEnv->pResultBuf;
766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788

  SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId);
  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // not assign result buffer yet, add new result buffer
  if (pResultRow->pageId == -1) {
    int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) tableGroupId, pRuntimeEnv->pQueryAttr->intermediateResultRowSize);
    if (ret != TSDB_CODE_SUCCESS) {
      return -1;
    }
  }

  // set time window for current result
  pResultRow->win = (*win);
  *pResult = pResultRow;
  setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset);

  return TSDB_CODE_SUCCESS;
}

H
Haojun Liao 已提交
789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819
static void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf * pBuf, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset);

static int32_t setResultOutputBufByKey_rv(SResultRowInfo *pResultRowInfo, int64_t id, STimeWindow *win,
                                       bool masterscan, SResultRow **pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
                                       int32_t numOfOutput, int32_t* rowCellInfoOffset, SDiskbasedBuf *pBuf, SAggSupporter *pAggSup, SExecTaskInfo* pTaskInfo) {
  assert(win->skey <= win->ekey);
  SResultRow *pResultRow = doSetResultOutBufByKey_rv(pResultRowInfo, id, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId,
       pTaskInfo, true, pAggSup);

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // not assign result buffer yet, add new result buffer
  if (pResultRow->pageId == -1) { // todo intermediate result size
    int32_t ret = addNewWindowResultBuf(pResultRow, pBuf, (int32_t) tableGroupId, 0);
    if (ret != TSDB_CODE_SUCCESS) {
      return -1;
    }
  }

  // set time window for current result
  pResultRow->win = (*win);
  *pResult = pResultRow;
  setResultRowOutputBufInitCtx_rv(pBuf, pResultRow, pCtx, numOfOutput, rowCellInfoOffset);

  return TSDB_CODE_SUCCESS;
}


820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp   = true;
  }
}

static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp   == true;
  }
}

static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos,
                                      int16_t order, int64_t *pData) {
  int32_t forwardStep = 0;

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*) &pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
      forwardStep = end;

      if (pData[end + pos] == ekey) {
        forwardStep += 1;
      }
    }
  } else {
    int32_t end = searchFn((char *)pData, pos + 1, ekey, order);
    if (end >= 0) {
      forwardStep = pos - end;

      if (pData[end] == ekey) {
        forwardStep += 1;
      }
    }
  }

  assert(forwardStep >= 0);
  return forwardStep;
}

static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) {
  int64_t skey = TSKEY_INITIAL_VAL;
  int32_t i = 0;
  for (i = pResultRowInfo->size - 1; i >= 0; --i) {
    SResultRow *pResult = pResultRowInfo->pResult[i];
    if (pResult->closed) {
      break;
    }

    // new closed result rows
    if (timeWindowInterpo) {
      if (pResult->endInterp && ((pResult->win.skey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery))) {
        if (i > 0) { // the first time window, the startInterp is false.
          assert(pResult->startInterp);
        }

        closeResultRow(pResultRowInfo, i);
      } else {
        skey = pResult->win.skey;
      }
    } else {
      if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) {
        closeResultRow(pResultRowInfo, i);
      } else {
        skey = pResult->win.skey;
      }
    }
  }

  // all result rows are closed, set the last one to be the skey
  if (skey == TSKEY_INITIAL_VAL) {
    if (pResultRowInfo->size == 0) {
//      assert(pResultRowInfo->current == NULL);
      assert(pResultRowInfo->curPos == -1);
      pResultRowInfo->curPos = -1;
    } else {
      pResultRowInfo->curPos = pResultRowInfo->size - 1;
    }
  } else {

    for (i = pResultRowInfo->size - 1; i >= 0; --i) {
      SResultRow *pResult = pResultRowInfo->pResult[i];
      if (pResult->closed) {
        break;
      }
    }

    if (i == pResultRowInfo->size - 1) {
      pResultRowInfo->curPos = i;
    } else {
      pResultRowInfo->curPos = i + 1;  // current not closed result object
    }
  }
}

H
Haojun Liao 已提交
921 922
static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, const STimeWindow* pWin, TSKEY lastKey, bool ascQuery, bool interp) {
  if ((lastKey > pWin->ekey && ascQuery) || (lastKey < pWin->ekey && (!ascQuery))) {
923 924 925 926
    closeAllResultRows(pResultRowInfo);
    pResultRowInfo->curPos = pResultRowInfo->size - 1;
  } else {
    int32_t step = ascQuery ? 1 : -1;
H
Haojun Liao 已提交
927
    doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, interp);
928 929 930
  }
}

H
Haojun Liao 已提交
931 932
static int32_t getNumOfRowsInTimeWindow(SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn,
                                        int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
933 934 935 936 937
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num   = -1;
  int32_t step  = GET_FORWARD_DIRECTION_FACTOR(order);

H
Haojun Liao 已提交
938
  if (order == TSDB_ORDER_ASC) {
939 940
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
H
Haojun Liao 已提交
941
      if (item != NULL) {
942 943 944 945
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
H
Haojun Liao 已提交
946
      if (item != NULL) {
947 948 949 950 951 952
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
H
Haojun Liao 已提交
953
      if (item != NULL) {
954 955 956 957
        item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step;
      }
    } else {
      num = startPos + 1;
H
Haojun Liao 已提交
958
      if (item != NULL) {
959 960 961 962 963 964 965 966 967
        item->lastKey = pDataBlockInfo->window.skey + step;
      }
    }
  }

  assert(num >= 0);
  return num;
}

H
Haojun Liao 已提交
968 969
static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, int32_t forwardStep, TSKEY* tsCol,
    int32_t numOfTotal, int32_t numOfOutput, int32_t order) {
970 971 972 973 974
  for (int32_t k = 0; k < numOfOutput; ++k) {
    pCtx[k].size    = forwardStep;
    pCtx[k].startTs = pWin->skey;

    // keep it temporarialy
H
Haojun Liao 已提交
975 976
    int32_t startOffset = pCtx[k].startRow;
    bool    hasAgg      = pCtx[k].isAggSet;
977

H
Haojun Liao 已提交
978 979
    int32_t pos = (order == TSDB_ORDER_ASC) ? offset : offset - (forwardStep - 1);
    pCtx[k].startRow = pos;
980 981 982 983 984 985 986 987 988 989 990

    if (tsCol != NULL) {
      pCtx[k].ptsList = &tsCol[pos];
    }

    // not a whole block involved in query processing, statistics data can not be used
    // NOTE: the original value of isSet have been changed here
    if (pCtx[k].isAggSet && forwardStep < numOfTotal) {
      pCtx[k].isAggSet = false;
    }

H
Haojun Liao 已提交
991
    if (functionNeedToExecute(&pCtx[k])) {
H
Haojun Liao 已提交
992
      pCtx[k].fpSet->addInput(&pCtx[k]);
993 994 995
    }

    // restore it
H
Haojun Liao 已提交
996 997
    pCtx[k].isAggSet = hasAgg;
    pCtx[k].startRow = startOffset;
998 999 1000
  }
}

H
Haojun Liao 已提交
1001 1002 1003 1004 1005 1006 1007
static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
                                      TSKEY* primaryKeys, int32_t prevPosition, STableIntervalOperatorInfo* pInfo) {
  int32_t order = pInfo->order;
  bool    ascQuery  = (order == TSDB_ORDER_ASC);

  int32_t precision = pInfo->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);
1008 1009

  // next time window is not in current block
H
Haojun Liao 已提交
1010 1011
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
1012 1013 1014
    return -1;
  }

H
Haojun Liao 已提交
1015
  TSKEY startKey = ascQuery? pNext->skey:pNext->ekey;
1016 1017 1018
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
H
Haojun Liao 已提交
1019 1020
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
    int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
1021 1022
    startPos = prevPosition + factor;
  } else {
H
Haojun Liao 已提交
1023
    if (startKey <= pDataBlockInfo->window.skey && ascQuery) {
1024
      startPos = 0;
H
Haojun Liao 已提交
1025
    } else if (startKey >= pDataBlockInfo->window.ekey && !ascQuery) {
1026 1027
      startPos = pDataBlockInfo->rows - 1;
    } else {
H
Haojun Liao 已提交
1028
      startPos = binarySearchForKey((char *)primaryKeys, pDataBlockInfo->rows, startKey, order);
1029 1030 1031 1032
    }
  }

  /* interp query with fill should not skip time window */
H
Haojun Liao 已提交
1033 1034 1035
//  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
//    return startPos;
//  }
1036 1037 1038 1039 1040 1041

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
H
Haojun Liao 已提交
1042
    if (ascQuery) {
1043 1044 1045 1046 1047
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
H
Haojun Liao 已提交
1048
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
1049
      TSKEY next = primaryKeys[startPos];
H
Haojun Liao 已提交
1050 1051 1052
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
1053
      } else {
H
Haojun Liao 已提交
1054 1055
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1)/pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
1056
      }
H
Haojun Liao 已提交
1057
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
1058
      TSKEY next = primaryKeys[startPos];
H
Haojun Liao 已提交
1059 1060 1061
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
1062
      } else {
H
Haojun Liao 已提交
1063 1064
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
1065 1066 1067 1068 1069 1070 1071
      }
    }
  }

  return startPos;
}

H
Haojun Liao 已提交
1072
static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr *pQueryAttr, STimeWindow *pWindow) {
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
  TSKEY ekey = -1;
  if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
    ekey = pWindow->ekey;
    if (ekey > pQueryAttr->window.ekey) {
      ekey = pQueryAttr->window.ekey;
    }
  } else {
    ekey = pWindow->skey;
    if (ekey < pQueryAttr->window.ekey) {
      ekey = pQueryAttr->window.ekey;
    }
  }

  return ekey;
}

H
Haojun Liao 已提交
1089
static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

H
Haojun Liao 已提交
1101
static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowIndex, int32_t numOfCols) {
1102 1103 1104 1105
  if (pDataBlock == NULL) {
    return;
  }

H
Haojun Liao 已提交
1106
  for (int32_t k = 0; k < numOfCols; ++k) {
1107
    SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k);
H
Haojun Liao 已提交
1108
    memcpy(pRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes);
1109 1110 1111
  }
}

H
Haojun Liao 已提交
1112
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, bool ascQuery) {
1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
  TSKEY ts = TSKEY_INITIAL_VAL;
  if (tsCols == NULL) {
    ts = ascQuery? win->skey : win->ekey;
  } else {
    int32_t offset = ascQuery? 0:rows-1;
    ts = tsCols[offset];
  }

  return ts;
}

H
Haojun Liao 已提交
1124
static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order);
1125

H
Haojun Liao 已提交
1126
static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) {
1127 1128 1129 1130 1131
  for (int32_t i = 0; i < pOperator->numOfOutput; ++i) {
    pCtx[i].order = order;
    pCtx[i].size  = pBlock->info.rows;
    pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag;

1132
    setBlockStatisInfo(&pCtx[i], pBlock, NULL/*&pOperator->pExpr[i].base.colInfo*/);
1133 1134 1135
  }
}

H
Haojun Liao 已提交
1136
void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) {
H
Haojun Liao 已提交
1137
//  if (pCtx[0].functionId == FUNCTION_ARITHM) {
1138 1139 1140 1141 1142 1143
//    SScalar* pSupport = (SScalarFunctionSupport*) pCtx[0].param[1].pz;
//    if (pSupport->colList == NULL) {
//      doSetInputDataBlock(pOperator, pCtx, pBlock, order);
//    } else {
//      doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order);
//    }
H
Haojun Liao 已提交
1144
//  } else {
1145 1146 1147 1148 1149
    if (pBlock->pDataBlock != NULL) {
      doSetInputDataBlock(pOperator, pCtx, pBlock, order);
    } else {
      doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order);
    }
H
Haojun Liao 已提交
1150
//  }
1151 1152
}

H
Haojun Liao 已提交
1153
static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) {
1154 1155 1156
  for (int32_t i = 0; i < pOperator->numOfOutput; ++i) {
    pCtx[i].order = order;
    pCtx[i].size  = pBlock->info.rows;
H
Haojun Liao 已提交
1157
    pCtx[i].currentStage = MAIN_SCAN/*(uint8_t)pOperator->pRuntimeEnv->scanFlag*/;
1158

H
Haojun Liao 已提交
1159
    setBlockStatisInfo(&pCtx[i], pBlock, pOperator->pExpr[i].base.pColumns);
1160 1161 1162 1163

    if (pCtx[i].functionId == FUNCTION_ARITHM) {
//      setArithParams((SScalarFunctionSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock);
    } else {
H
Haojun Liao 已提交
1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177
      uint32_t flag = pOperator->pExpr[i].base.pColumns->flag;
      if (TSDB_COL_IS_NORMAL_COL(flag) /*|| (pCtx[i].functionId == FUNCTION_BLKINFO) ||
          (TSDB_COL_IS_TAG(flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)*/) {

        SColumn* pCol = pOperator->pExpr[i].base.pColumns;
        if (pCtx[i].columnIndex == -1) {
          for(int32_t j = 0; j < pBlock->info.numOfCols; ++j) {
            SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j);
            if (pColData->info.colId == pCol->info.colId) {
              pCtx[i].columnIndex = j;
              break;
            }
          }
        }
1178

H
Haojun Liao 已提交
1179
        SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pCtx[i].columnIndex);
1180
        // in case of the block distribution query, the inputBytes is not a constant value.
1181
        pCtx[i].pInput = p;
H
Haojun Liao 已提交
1182
        assert(p->info.colId == pCol->info.colId);
1183 1184 1185 1186 1187 1188 1189 1190 1191 1192

        if (pCtx[i].functionId < 0) {
          SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0);
          pCtx[i].ptsList = (int64_t*) tsInfo->pData;

          continue;
        }

//        uint32_t status = aAggs[pCtx[i].functionId].status;
//        if ((status & (FUNCSTATE_SELECTIVITY | FUNCSTATE_NEED_TS)) != 0) {
1193
//          SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0);
H
Haojun Liao 已提交
1194 1195
          // In case of the top/bottom query again the nest query result, which has no timestamp column
          // don't set the ptsList attribute.
1196 1197 1198 1199 1200
//          if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
//            pCtx[i].ptsList = (int64_t*) tsInfo->pData;
//          } else {
//            pCtx[i].ptsList = NULL;
//          }
H
Haojun Liao 已提交
1201 1202 1203 1204 1205 1206 1207 1208 1209 1210
//        }
//      } else if (TSDB_COL_IS_UD_COL(pCol->flag) && (pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) {
//        SColIndex*       pColIndex = &pOperator->pExpr[i].base.colInfo;
//        SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex);
//
//        pCtx[i].pInput = p->pData;
//        assert(p->info.colId == pColIndex->info.colId && pCtx[i].inputType == p->info.type);
//        for(int32_t j = 0; j < pBlock->info.rows; ++j) {
//          char* dst = p->pData + j * p->info.bytes;
//          taosVariantDump(&pOperator->pExpr[i].base.param[1], dst, p->info.type, true);
1211 1212 1213 1214 1215 1216
//        }
      }
    }
  }
}

H
Haojun Liao 已提交
1217
static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunctionCtx* pCtx, SSDataBlock* pSDataBlock) {
1218
  for (int32_t k = 0; k < pOperator->numOfOutput; ++k) {
H
Haojun Liao 已提交
1219
    if (functionNeedToExecute(&pCtx[k])) {
1220
      pCtx[k].startTs = startTs;// this can be set during create the struct
H
Haojun Liao 已提交
1221
      pCtx[k].fpSet->addInput(&pCtx[k]);
1222 1223 1224 1225
    }
  }
}

H
Haojun Liao 已提交
1226
static void projectApplyFunctions(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx, int32_t numOfOutput) {
H
Haojun Liao 已提交
1227
  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250

  for (int32_t k = 0; k < numOfOutput; ++k) {
    pCtx[k].startTs = pQueryAttr->window.skey;

    // Always set the asc order for merge stage process
    if (pCtx[k].currentStage == MERGE_STAGE) {
      pCtx[k].order = TSDB_ORDER_ASC;
    }

    pCtx[k].startTs = pQueryAttr->window.skey;

    if (pCtx[k].functionId < 0) {
      // load the script and exec
//      SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo;
//      doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL);
//    } else {
//      aAggs[pCtx[k].functionId].xFunction(&pCtx[k]);
    }
  }
}

void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs,
                               int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) {
H
Haojun Liao 已提交
1251
  STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv;
1252 1253
  SExprInfo* pExpr = pOperator->pExpr;

H
Haojun Liao 已提交
1254
  SqlFunctionCtx* pCtx = pInfo->pCtx;
1255 1256 1257 1258 1259 1260 1261 1262

  for (int32_t k = 0; k < pOperator->numOfOutput; ++k) {
    int32_t functionId = pCtx[k].functionId;
    if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) {
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

1263
    SColIndex *      pColIndex = NULL/*&pExpr[k].base.colInfo*/;
1264 1265 1266
    int16_t          index = pColIndex->colIndex;
    SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, index);

1267
//    assert(pColInfo->info.colId == pColIndex->info.colId && curTs != windowKey);
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
    double v1 = 0, v2 = 0, v = 0;

    if (prevRowIndex == -1) {
      GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]);
    } else {
      GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes);
    }

    GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes);

    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            pCtx[k].start.ptr = (char *)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
      SPoint point1 = (SPoint){.key = prevTs,    .val = &v1};
      SPoint point2 = (SPoint){.key = curTs,     .val = &v2};
      SPoint point  = (SPoint){.key = windowKey, .val = &v };

      taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);

      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = point.key;
        pCtx[k].start.val = v;
      } else {
        pCtx[k].end.key = point.key;
        pCtx[k].end.val = v;
      }
    }
  }
}

H
Haojun Liao 已提交
1314
static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos,
1315
                                              int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) {
H
Haojun Liao 已提交
1316 1317
  STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv;
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344

  bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr);

  TSKEY curTs  = tsCols[pos];
  TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0];

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery? win->skey:win->ekey;
  if (key == curTs) {
    setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP);
    return true;
  }

  if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) {
    setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP);
    return true;
  }

  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
  TSKEY   prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))? lastTs:tsCols[pos - step];

  doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos,
      key, RESULT_ROW_START_INTERP);
  return true;
}

H
Haojun Liao 已提交
1345
static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx,
1346
    int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) {
H
Haojun Liao 已提交
1347 1348
  STaskRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv;
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376
  int32_t numOfOutput = pOperatorInfo->numOfOutput;

  TSKEY   actualEndKey = tsCols[endRowIndex];

  TSKEY key = QUERY_IS_ASC_QUERY(pQueryAttr)? win->ekey:win->skey;

  // not ended in current data block, do not invoke interpolation
  if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQueryAttr)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQueryAttr))) {
    setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP);
    return false;
  }

  // there is actual end point of current time window, no interpolation need
  if (key == actualEndKey) {
    setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP);
    return true;
  }

  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
  int32_t nextRowIndex = endRowIndex + step;
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
  doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey,
      nextRowIndex, key, RESULT_ROW_END_INTERP);
  return true;
}

H
Haojun Liao 已提交
1377
static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx,
H
Haojun Liao 已提交
1378 1379
    SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep, int32_t order, bool timeWindowInterpo) {
  if (!timeWindowInterpo) {
1380 1381 1382 1383
    return;
  }

  assert(pBlock != NULL);
H
Haojun Liao 已提交
1384
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);
1385 1386 1387 1388 1389

  if (pBlock->pDataBlock == NULL){
//    tscError("pBlock->pDataBlock == NULL");
    return;
  }
H
Haojun Liao 已提交
1390

1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402
  SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0);

  TSKEY  *tsCols = (TSKEY *)(pColInfo->pData);
  bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
  if (!done) { // it is not interpolated, now start to generated the interpolated value
    int32_t startRowIndex = startPos;
    bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, pBlock->pDataBlock,
        tsCols, win);
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
H
Haojun Liao 已提交
1403
    setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP);
1404 1405 1406
  }

  // point interpolation does not require the end key time window interpolation.
H
Haojun Liao 已提交
1407 1408 1409
//  if (pointInterpQuery) {
//    return;
//  }
1410 1411 1412 1413 1414 1415

  // interpolation query does not generate the time window end interpolation
  done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
  if (!done) {
    int32_t endRowIndex = startPos + (forwardStep - 1) * step;

H
Haojun Liao 已提交
1416
    TSKEY endKey = (order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey;
1417 1418 1419 1420 1421
    bool  interp = setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
H
Haojun Liao 已提交
1422
    setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_END_INTERP);
1423 1424 1425 1426 1427 1428
  }
}

static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) {
  STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info;

H
Haojun Liao 已提交
1429 1430
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  int32_t numOfOutput = pOperatorInfo->numOfOutput;
1431

H
Haojun Liao 已提交
1432 1433
  int32_t step = 1;
  bool ascQuery = true;
1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445

  int32_t prevIndex = pResultRowInfo->curPos;

  TSKEY* tsCols = NULL;
  if (pSDataBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0);
    tsCols = (int64_t*) pColDataInfo->pData;
    assert(tsCols[0] == pSDataBlock->info.window.skey &&
           tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey);
  }

  int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1);
H
Haojun Liao 已提交
1446
  TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascQuery);
1447

H
Haojun Liao 已提交
1448 1449
  STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, &pInfo->interval, pInfo->precision, &pInfo->win);
  bool masterScan = true;
1450 1451

  SResultRow* pResult = NULL;
H
Haojun Liao 已提交
1452 1453
  int32_t ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx,
                                        numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo);
1454
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
H
Haojun Liao 已提交
1455
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
1456 1457 1458
  }

  int32_t forwardStep = 0;
H
Haojun Liao 已提交
1459
  TSKEY   ekey = win.ekey;
1460
  forwardStep =
H
Haojun Liao 已提交
1461
      getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
1462 1463 1464

  // prev time window not interpolation yet.
  int32_t curIndex = pResultRowInfo->curPos;
H
Haojun Liao 已提交
1465
  if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) {
1466 1467 1468 1469 1470 1471 1472 1473
    for (int32_t j = prevIndex; j < curIndex; ++j) {  // previous time window may be all closed already.
      SResultRow* pRes = getResultRow(pResultRowInfo, j);
      if (pRes->closed) {
        assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP));
        continue;
      }

        STimeWindow w = pRes->win;
H
Haojun Liao 已提交
1474 1475
        ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult,
                                      tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo);
1476
        if (ret != TSDB_CODE_SUCCESS) {
H
Haojun Liao 已提交
1477
          longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
1478 1479 1480 1481
        }

        assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP));

H
Haojun Liao 已提交
1482
        doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pSDataBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1,
1483 1484 1485
                                  tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP);

        setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
H
Haojun Liao 已提交
1486
        setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP);
1487

H
Haojun Liao 已提交
1488
        doApplyFunctions(pInfo->binfo.pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
1489 1490 1491
      }

    // restore current time window
H
Haojun Liao 已提交
1492 1493
    ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx,
                                  numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo);
1494
    if (ret != TSDB_CODE_SUCCESS) {
H
Haojun Liao 已提交
1495
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
1496 1497 1498 1499
    }
  }

  // window start key interpolation
H
Haojun Liao 已提交
1500 1501
  doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, pInfo->order, false);
  doApplyFunctions(pInfo->binfo.pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
1502 1503 1504 1505

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = (forwardStep - 1) * step + startPos;
H
Haojun Liao 已提交
1506
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo);
1507 1508 1509 1510 1511
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
H
Haojun Liao 已提交
1512 1513
    int32_t code = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &nextWin, masterScan, &pResult, tableGroupId,
                                           pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo);
1514
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
H
Haojun Liao 已提交
1515
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
1516 1517
    }

H
Haojun Liao 已提交
1518 1519
    ekey = nextWin.ekey;//reviseWindowEkey(pQueryAttr, &nextWin);
    forwardStep = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
1520 1521

    // window start(end) key interpolation
H
Haojun Liao 已提交
1522 1523
    doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, pInfo->order, false);
    doApplyFunctions(pInfo->binfo.pCtx, &nextWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
1524 1525
  }

H
Haojun Liao 已提交
1526
  if (pInfo->timeWindowInterpo) {
1527
    int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0;
H
Haojun Liao 已提交
1528
    saveDataBlockLastRow(pInfo->pRow, pSDataBlock->pDataBlock, rowIndex, pSDataBlock->info.numOfCols);
1529 1530
  }

H
Haojun Liao 已提交
1531
//  updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false);
1532 1533 1534 1535 1536 1537
}


static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) {
  STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info;

H
Haojun Liao 已提交
1538
  STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv;
1539
  int32_t           numOfOutput = pOperatorInfo->numOfOutput;
H
Haojun Liao 已提交
1540
  STaskAttr*       pQueryAttr = pRuntimeEnv->pQueryAttr;
1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553

  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
  bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr);

  TSKEY* tsCols = NULL;
  if (pSDataBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0);
    tsCols = (int64_t*) pColDataInfo->pData;
    assert(tsCols[0] == pSDataBlock->info.window.skey &&
           tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey);
  }

  int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1);
H
Haojun Liao 已提交
1554
  TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascQuery);
1555 1556

  STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr);
H
Haojun Liao 已提交
1557
  bool masterScan = IS_MAIN_SCAN(pRuntimeEnv);
1558 1559 1560 1561 1562 1563 1564 1565 1566

  SResultRow* pResult = NULL;
  int32_t forwardStep = 0;
  int32_t ret = 0;
  STimeWindow preWin = win;

  while (1) {
    // null data, failed to allocate more memory buffer
    ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult,
H
Haojun Liao 已提交
1567
                                  tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset);
1568 1569 1570 1571 1572
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    TSKEY   ekey = reviseWindowEkey(pQueryAttr, &win);
H
Haojun Liao 已提交
1573
//    forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true);
1574 1575

    // window start(end) key interpolation
H
Haojun Liao 已提交
1576 1577
//    doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep);
//    doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput);
1578 1579 1580
    preWin = win;

    int32_t prevEndPos = (forwardStep - 1) * step + startPos;
H
Haojun Liao 已提交
1581
//    startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos);
1582 1583 1584
    if (startPos < 0) {
      if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) {
        int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId,
H
Haojun Liao 已提交
1585
                                               pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset);
1586 1587 1588 1589 1590 1591 1592
        if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
          longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
        }

        startPos = pSDataBlock->info.rows - 1;

        // window start(end) key interpolation
H
Haojun Liao 已提交
1593 1594
//        doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep);
//        doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput);
1595 1596 1597 1598 1599 1600 1601 1602 1603
      }

      break;
    }
    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
  }

  if (pQueryAttr->timeWindowInterpo) {
    int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0;
H
Haojun Liao 已提交
1604
//    saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex);
1605 1606
  }

H
Haojun Liao 已提交
1607
//  updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey);
1608 1609 1610 1611 1612
}



static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) {
H
Haojun Liao 已提交
1613
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
1614 1615 1616 1617
  STableQueryInfo*  item = pRuntimeEnv->current;

  SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex);

H
Haojun Liao 已提交
1618
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
1619 1620 1621 1622
  int16_t     bytes = pColInfoData->info.bytes;
  int16_t     type = pColInfoData->info.type;

  if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) {
1623
    //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv));
1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668
    return;
  }

  SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0);
  int64_t* tsList = (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (int64_t*) pFirstColData->pData:NULL;

  STimeWindow w = TSWINDOW_INITIALIZER;

  int32_t num = 0;
  for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) {
    char* val = ((char*)pColInfoData->pData) + bytes * j;
    if (isNull(val, type)) {
      continue;
    }

    // Compare with the previous row of this column, and do not set the output buffer again if they are identical.
    if (pInfo->prevData == NULL) {
      pInfo->prevData = malloc(bytes);
      memcpy(pInfo->prevData, val, bytes);
      num++;
      continue;
    }

    if (IS_VAR_DATA_TYPE(type)) {
      int32_t len = varDataLen(val);
      if(len == varDataLen(pInfo->prevData) && memcmp(varDataVal(pInfo->prevData), varDataVal(val), len) == 0) {
        num++;
        continue;
      }
    } else {
      if (memcmp(pInfo->prevData, val, bytes) == 0) {
        num++;
        continue;
      }
    }

    if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) {
      setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, bytes);
    }

    int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, item->groupIndex);
    if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
      longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
    }

H
Haojun Liao 已提交
1669
//    doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput);
1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687

    num = 1;
    memcpy(pInfo->prevData, val, bytes);
  }

  if (num > 0) {
    char* val = ((char*)pColInfoData->pData) + bytes * (pSDataBlock->info.rows - num);
    memcpy(pInfo->prevData, val, bytes);

    if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) {
      setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes);
    }

    int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex);
    if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
      longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
    }

H
Haojun Liao 已提交
1688
//    doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput);
1689 1690 1691 1692 1693 1694
  }

  tfree(pInfo->prevData);
}

static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) {
H
Haojun Liao 已提交
1695
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
1696 1697 1698 1699 1700
  STableQueryInfo*  item = pRuntimeEnv->current;

  // primary timestamp column
  SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0);

H
Haojun Liao 已提交
1701
  bool    masterScan = IS_MAIN_SCAN(pRuntimeEnv);
1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737
  SOptrBasicInfo* pBInfo = &pInfo->binfo;

  int64_t gap = pOperator->pRuntimeEnv->pQueryAttr->sw.gap;
  pInfo->numOfRows = 0;
  if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->prevTs = INT64_MIN;
  }

  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) {
    if (pInfo->prevTs == INT64_MIN) {
      pInfo->curWindow.skey = tsList[j];
      pInfo->curWindow.ekey = tsList[j];
      pInfo->prevTs = tsList[j];
      pInfo->numOfRows = 1;
      pInfo->start = j;
    } else if (tsList[j] - pInfo->prevTs <= gap && (tsList[j] - pInfo->prevTs) >= 0) {
      pInfo->curWindow.ekey = tsList[j];
      pInfo->prevTs = tsList[j];
      pInfo->numOfRows += 1;
      if (j == 0 && pInfo->start != 0) {
        pInfo->numOfRows = 1;
        pInfo->start = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      pInfo->curWindow.ekey = pInfo->curWindow.skey;
      int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan,
                                            &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput,
                                            pBInfo->rowCellInfoOffset);
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
      }

H
Haojun Liao 已提交
1738 1739
//      doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList,
//                       pSDataBlock->info.rows, pOperator->numOfOutput);
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758

      pInfo->curWindow.skey = tsList[j];
      pInfo->curWindow.ekey = tsList[j];
      pInfo->prevTs = tsList[j];
      pInfo->numOfRows = 1;
      pInfo->start = j;
    }
  }

  SResultRow* pResult = NULL;

  pInfo->curWindow.ekey = pInfo->curWindow.skey;
  int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan,
                                        &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput,
                                        pBInfo->rowCellInfoOffset);
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
  }

H
Haojun Liao 已提交
1759 1760
//  doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList,
//                   pSDataBlock->info.rows, pOperator->numOfOutput);
1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779
}

static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) {
  if (IS_VAR_DATA_TYPE(type)) {
    if (pResultRow->key == NULL) {
      pResultRow->key = malloc(varDataTLen(pData));
      varDataCopy(pResultRow->key, pData);
    } else {
      assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0);
    }
  } else {
    int64_t v = -1;
    GET_TYPED_DATA(v, int64_t, type, pData);

    pResultRow->win.skey = v;
    pResultRow->win.ekey = v;
  }
}

H
Haojun Liao 已提交
1780
static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) {
1781
  SDiskbasedBuf *pResultBuf = pRuntimeEnv->pResultBuf;
1782 1783 1784

  int32_t        *rowCellInfoOffset = binfo->rowCellInfoOffset;
  SResultRowInfo *pResultRowInfo    = &binfo->resultRowInfo;
H
Haojun Liao 已提交
1785
  SqlFunctionCtx *pCtx              = binfo->pCtx;
1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833

  // not assign result buffer yet, add new result buffer, TODO remove it
  char* d = pData;
  int16_t len = bytes;
  if (IS_VAR_DATA_TYPE(type)) {
    d = varDataVal(pData);
    len = varDataLen(pData);
  }

  int64_t tid = 0;
  SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, d, len, true, groupIndex);
  assert (pResultRow != NULL);

  setResultRowKey(pResultRow, pData, type);
  if (pResultRow->pageId == -1) {
    int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->pQueryAttr->resultRowSize);
    if (ret != 0) {
      return -1;
    }
  }

  setResultOutputBuf(pRuntimeEnv, pResultRow, pCtx, numOfCols, rowCellInfoOffset);
  initCtxOutputBuffer(pCtx, numOfCols);
  return TSDB_CODE_SUCCESS;
}

static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock) {
  size_t num = taosArrayGetSize(pGroupbyExpr->columnInfo);
  for (int32_t k = 0; k < num; ++k) {
    SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k);
    if (TSDB_COL_IS_TAG(pColIndex->flag)) {
      continue;
    }

    int32_t colId = pColIndex->colId;

    for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
      SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i);
      if (pColInfo->info.colId == colId) {
        return i;
      }
    }
  }

  assert(0);
  return -1;
}

H
Haojun Liao 已提交
1834
static bool functionNeedToExecute(SqlFunctionCtx *pCtx) {
1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847
  struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);

  // in case of timestamp column, always generated results.
  int32_t functionId = pCtx->functionId;
  if (functionId == FUNCTION_TS) {
    return true;
  }

  if (isRowEntryCompleted(pResInfo) || functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) {
    return false;
  }

  if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) {
H
Haojun Liao 已提交
1848
//    return QUERY_IS_ASC_QUERY(pQueryAttr);
1849 1850 1851 1852
  }

  // denote the order type
  if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) {
H
Haojun Liao 已提交
1853
//    return pCtx->param[0].i == pQueryAttr->order.order;
1854 1855 1856
  }

  // in the reverse table scan, only the following functions need to be executed
H
Haojun Liao 已提交
1857 1858 1859 1860
//  if (IS_REVERSE_SCAN(pRuntimeEnv) ||
//      (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != FUNCTION_STDDEV && functionId != FUNCTION_PERCT)) {
//    return false;
//  }
1861 1862 1863 1864

  return true;
}

H
Haojun Liao 已提交
1865
void setBlockStatisInfo(SqlFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn) {
1866 1867
  SColumnDataAgg *pAgg = NULL;

H
Haojun Liao 已提交
1868 1869
  if (pSDataBlock->pBlockAgg != NULL && TSDB_COL_IS_NORMAL_COL(pColumn->flag)) {
    pAgg = &pSDataBlock->pBlockAgg[pCtx->columnIndex];
1870 1871 1872 1873 1874 1875 1876 1877

    pCtx->agg = *pAgg;
    pCtx->isAggSet  = true;
    assert(pCtx->agg.numOfNull <= pSDataBlock->info.rows);
  } else {
    pCtx->isAggSet = false;
  }

H
Haojun Liao 已提交
1878
  pCtx->hasNull = hasNull(pColumn, pAgg);
1879 1880

  // set the statistics data for primary time stamp column
H
Haojun Liao 已提交
1881
  if (pCtx->functionId == FUNCTION_SPREAD && pColumn->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
1882 1883 1884 1885 1886 1887 1888
    pCtx->isAggSet  = true;
    pCtx->agg.min = pSDataBlock->info.window.skey;
    pCtx->agg.max = pSDataBlock->info.window.ekey;
  }
}

// set the output buffer for the selectivity + tag query
H
Haojun Liao 已提交
1889
static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) {
1890 1891 1892 1893 1894 1895 1896
  if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) {
    return TSDB_CODE_SUCCESS;
  }

  int32_t num = 0;
  int16_t tagLen = 0;

H
Haojun Liao 已提交
1897 1898
  SqlFunctionCtx*  p = NULL;
  SqlFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES);
1899 1900 1901 1902 1903 1904 1905 1906
  if (pTagCtx == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }

  for (int32_t i = 0; i < numOfOutput; ++i) {
    int32_t functionId = pCtx[i].functionId;

    if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) {
H
Haojun Liao 已提交
1907
      tagLen += pCtx[i].resDataInfo.bytes;
1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929
      pTagCtx[num++] = &pCtx[i];
    } else if (1/*(aAggs[functionId].status & FUNCSTATE_SELECTIVITY) != 0*/) {
      p = &pCtx[i];
    } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) {
      // tag function may be the group by tag column
      // ts may be the required primary timestamp column
      continue;
    } else {
      // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ
    }
  }
  if (p != NULL) {
    p->tagInfo.pTagCtxList = pTagCtx;
    p->tagInfo.numOfTagCols = num;
    p->tagInfo.tagsLen = tagLen;
  } else {
    tfree(pTagCtx);
  }

  return TSDB_CODE_SUCCESS;
}

H
Haojun Liao 已提交
1930
static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput,
1931
                                            int32_t** rowCellInfoOffset) {
H
Haojun Liao 已提交
1932
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
1933

H
Haojun Liao 已提交
1934
  SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx));
1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946
  if (pFuncCtx == NULL) {
    return NULL;
  }

  *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t));
  if (*rowCellInfoOffset == 0) {
    tfree(pFuncCtx);
    return NULL;
  }

  for (int32_t i = 0; i < numOfOutput; ++i) {
    SSqlExpr *pSqlExpr = &pExpr[i].base;
H
Haojun Liao 已提交
1947
    SqlFunctionCtx* pCtx = &pFuncCtx[i];
1948
#if 0
1949 1950 1951 1952 1953 1954 1955 1956
    SColIndex *pIndex = &pSqlExpr->colInfo;

    if (TSDB_COL_REQ_NULL(pIndex->flag)) {
      pCtx->requireNull = true;
      pIndex->flag &= ~(TSDB_COL_NULL);
    } else {
      pCtx->requireNull = false;
    }
1957
#endif
1958 1959 1960 1961 1962
//    pCtx->inputBytes = pSqlExpr->colBytes;
//    pCtx->inputType  = pSqlExpr->colType;

    pCtx->ptsOutputBuf = NULL;

H
Haojun Liao 已提交
1963 1964
    pCtx->resDataInfo.bytes  = pSqlExpr->resSchema.bytes;
    pCtx->resDataInfo.type   = pSqlExpr->resSchema.type;
1965 1966 1967 1968

    pCtx->order        = pQueryAttr->order.order;
//    pCtx->functionId   = pSqlExpr->functionId;
    pCtx->stableQuery  = pQueryAttr->stableQuery;
H
Haojun Liao 已提交
1969
    pCtx->resDataInfo.intermediateBytes = pSqlExpr->interBytes;
1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
    pCtx->start.key    = INT64_MIN;
    pCtx->end.key      = INT64_MIN;

    pCtx->numOfParams  = pSqlExpr->numOfParams;
    for (int32_t j = 0; j < pCtx->numOfParams; ++j) {
      int16_t type = pSqlExpr->param[j].nType;
      int16_t bytes = pSqlExpr->param[j].nLen;
//      if (pSqlExpr->functionId == FUNCTION_STDDEV_DST) {
//        continue;
//      }

      if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
        taosVariantCreateFromBinary(&pCtx->param[j], pSqlExpr->param[j].pz, bytes, type);
      } else {
        taosVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlExpr->param[j].i, bytes, type);
      }
    }

    // set the order information for top/bottom query
    int32_t functionId = pCtx->functionId;

    if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
      int32_t f = getExprFunctionId(&pExpr[0]);
      assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY);

      pCtx->param[2].i = pQueryAttr->order.order;
      pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
      pCtx->param[3].i = functionId;
      pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT;

H
Haojun Liao 已提交
2000
      pCtx->param[1].i = pQueryAttr->order.col.info.colId;
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020
    } else if (functionId == FUNCTION_INTERP) {
      pCtx->param[2].i = (int8_t)pQueryAttr->fillType;
      if (pQueryAttr->fillVal != NULL) {
        if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) {
          pCtx->param[1].nType = TSDB_DATA_TYPE_NULL;
        } else {  // todo refactor, taosVariantCreateFromBinary should handle the NULL value
          if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) {
            taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType);
          }
        }
      }
    } else if (functionId == FUNCTION_TS_COMP) {
      pCtx->param[0].i = pQueryAttr->vgId;  //TODO this should be the parameter from client
      pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT;
    } else if (functionId == FUNCTION_TWA) {
      pCtx->param[1].i = pQueryAttr->window.skey;
      pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT;
      pCtx->param[2].i = pQueryAttr->window.ekey;
      pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
    } else if (functionId == FUNCTION_ARITHM) {
2021
//      pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i);
2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033
    }
  }

//  for(int32_t i = 1; i < numOfOutput; ++i) {
//    (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr[i - 1].base.interBytes);
//  }

  setCtxTagColumnInfo(pFuncCtx, numOfOutput);

  return pFuncCtx;
}

H
Haojun Liao 已提交
2034
static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset, uint32_t* pRowSize) {
H
Haojun Liao 已提交
2035 2036
  size_t numOfOutput = taosArrayGetSize(pExprInfo);

H
Haojun Liao 已提交
2037
  SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx));
H
Haojun Liao 已提交
2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048
  if (pFuncCtx == NULL) {
    return NULL;
  }

  *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t));
  if (*rowCellInfoOffset == 0) {
    tfree(pFuncCtx);
    return NULL;
  }

  for (int32_t i = 0; i < numOfOutput; ++i) {
H
Haojun Liao 已提交
2049 2050 2051
    SExprInfo* pExpr = taosArrayGetP(pExprInfo, i);

    SSqlExpr *pSqlExpr = &pExpr->base;
H
Haojun Liao 已提交
2052
    SqlFunctionCtx* pCtx = &pFuncCtx[i];
H
Haojun Liao 已提交
2053

H
Haojun Liao 已提交
2054 2055 2056 2057 2058 2059 2060 2061 2062 2063
#if 0
    SColIndex *pIndex = &pSqlExpr->colInfo;

    if (TSDB_COL_REQ_NULL(pIndex->flag)) {
      pCtx->requireNull = true;
      pIndex->flag &= ~(TSDB_COL_NULL);
    } else {
      pCtx->requireNull = false;
    }
#endif
H
Haojun Liao 已提交
2064
//    pCtx->inputBytes = pSqlExpr->;
H
Haojun Liao 已提交
2065 2066 2067
//    pCtx->inputType  = pSqlExpr->colType;

    pCtx->ptsOutputBuf = NULL;
H
Haojun Liao 已提交
2068
    pCtx->fpSet = fpSet;
H
Haojun Liao 已提交
2069
    pCtx->columnIndex = -1;
H
Haojun Liao 已提交
2070 2071 2072
    pCtx->resDataInfo.bytes  = pSqlExpr->resSchema.bytes;
    pCtx->resDataInfo.type   = pSqlExpr->resSchema.type;

H
Haojun Liao 已提交
2073
    pCtx->order        = TSDB_ORDER_ASC;
H
Haojun Liao 已提交
2074 2075 2076 2077
    if (i == 0) {
      pCtx->functionId = FUNCTION_TS;
    }

H
Haojun Liao 已提交
2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132
//    pCtx->functionId   = pSqlExpr->functionId;
//    pCtx->stableQuery  = pQueryAttr->stableQuery;
    pCtx->resDataInfo.intermediateBytes = pSqlExpr->interBytes;
    pCtx->start.key    = INT64_MIN;
    pCtx->end.key      = INT64_MIN;

    pCtx->numOfParams  = pSqlExpr->numOfParams;
    for (int32_t j = 0; j < pCtx->numOfParams; ++j) {
      int16_t type = pSqlExpr->param[j].nType;
      int16_t bytes = pSqlExpr->param[j].nLen;

      if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
        taosVariantCreateFromBinary(&pCtx->param[j], pSqlExpr->param[j].pz, bytes, type);
      } else {
        taosVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlExpr->param[j].i, bytes, type);
      }
    }

    // set the order information for top/bottom query
    int32_t functionId = pCtx->functionId;

    if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
      int32_t f = getExprFunctionId(&pExpr[0]);
      assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY);

//      pCtx->param[2].i = pQueryAttr->order.order;
      pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
      pCtx->param[3].i = functionId;
      pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT;

//      pCtx->param[1].i = pQueryAttr->order.col.info.colId;
    } else if (functionId == FUNCTION_INTERP) {
//      pCtx->param[2].i = (int8_t)pQueryAttr->fillType;
//      if (pQueryAttr->fillVal != NULL) {
//        if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) {
//          pCtx->param[1].nType = TSDB_DATA_TYPE_NULL;
//        } else {  // todo refactor, taosVariantCreateFromBinary should handle the NULL value
//          if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) {
//            taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType);
//          }
//        }
//      }
    } else if (functionId == FUNCTION_TS_COMP) {
//      pCtx->param[0].i = pQueryAttr->vgId;  //TODO this should be the parameter from client
      pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT;
    } else if (functionId == FUNCTION_TWA) {
//      pCtx->param[1].i = pQueryAttr->window.skey;
      pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT;
//      pCtx->param[2].i = pQueryAttr->window.ekey;
      pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
    } else if (functionId == FUNCTION_ARITHM) {
//      pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i);
    }
  }

H
Haojun Liao 已提交
2133 2134 2135
  for(int32_t i = 1; i < numOfOutput; ++i) {
    SExprInfo* pExpr = taosArrayGetP(pExprInfo, i - 1);
    (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr->base.interBytes);
H
Haojun Liao 已提交
2136
    *pRowSize += pExpr->base.resSchema.bytes;
H
Haojun Liao 已提交
2137
  }
H
Haojun Liao 已提交
2138 2139 2140 2141 2142

  setCtxTagColumnInfo(pFuncCtx, numOfOutput);
  return pFuncCtx;
}

H
Haojun Liao 已提交
2143
static void* destroySQLFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160
  if (pCtx == NULL) {
    return NULL;
  }

  for (int32_t i = 0; i < numOfOutput; ++i) {
    for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) {
      taosVariantDestroy(&pCtx[i].param[j]);
    }

    taosVariantDestroy(&pCtx[i].tag);
    tfree(pCtx[i].tagInfo.pTagCtxList);
  }

  tfree(pCtx);
  return NULL;
}

H
Haojun Liao 已提交
2161
static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) {
2162
  //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_TASKID(pRuntimeEnv));
H
Haojun Liao 已提交
2163
  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
2164 2165 2166 2167 2168 2169 2170

  pRuntimeEnv->prevGroupId = INT32_MIN;
  pRuntimeEnv->pQueryAttr = pQueryAttr;

  pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
  pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
  pRuntimeEnv->keyBuf  = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES);
H
Haojun Liao 已提交
2171
//  pRuntimeEnv->pool    = initResultRowPool(getResultRowSize(pRuntimeEnv));
2172 2173 2174 2175 2176 2177 2178 2179
  pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell));

  pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize);
  pRuntimeEnv->tagVal  = malloc(pQueryAttr->tagLen);

  // NOTE: pTableCheckInfo need to update the query time range and the lastKey info
  pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);

D
dapan1121 已提交
2180
  //pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput);
2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198

  if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL ||
      pRuntimeEnv->prevRow == NULL  || pRuntimeEnv->tagVal == NULL) {
    goto _clean;
  }

  if (pQueryAttr->numOfCols) {
    char* start = POINTER_BYTES * pQueryAttr->numOfCols + (char*) pRuntimeEnv->prevRow;
    pRuntimeEnv->prevRow[0] = start;
    for(int32_t i = 1; i < pQueryAttr->numOfCols; ++i) {
      pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQueryAttr->tableCols[i-1].bytes;
    }

    if (pQueryAttr->tableCols[0].type == TSDB_DATA_TYPE_TIMESTAMP) {
      *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN;
    }
  }

2199
  //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_TASKID(pRuntimeEnv));
2200 2201 2202 2203 2204 2205

  // group by normal column, sliding window query, interval query are handled by interval query processor
  // interval (down sampling operation)
  return TSDB_CODE_SUCCESS;

_clean:
D
dapan1121 已提交
2206
  //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput);
2207 2208 2209 2210 2211 2212 2213 2214
  tfree(pRuntimeEnv->pResultRowHashTable);
  tfree(pRuntimeEnv->keyBuf);
  tfree(pRuntimeEnv->prevRow);
  tfree(pRuntimeEnv->tagVal);

  return TSDB_CODE_QRY_OUT_OF_MEMORY;
}

H
Haojun Liao 已提交
2215 2216
static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) {
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
2217

2218
//  tsdbCleanupReadHandle(pRuntimeEnv->pTsdbReadHandle);
2219
  pRuntimeEnv->pTsdbReadHandle = NULL;
2220 2221 2222 2223 2224

//  SMemRef* pMemRef = &pQueryAttr->memRef;
//  assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL);
}

H
Haojun Liao 已提交
2225
static void destroyTsComp(STaskRuntimeEnv *pRuntimeEnv, STaskAttr *pQueryAttr) {
2226 2227 2228
  if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) {
    SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0);
    if (pColInfoData) {
2229 2230 2231 2232
      TdFilePtr pFile = *(TdFilePtr *)pColInfoData->pData;  // TODO refactor
      if (pFile != NULL) {
        taosCloseFile(&pFile);
        *(TdFilePtr *)pColInfoData->pData = NULL;
2233 2234 2235 2236 2237
      }
    }
  }
}

H
Haojun Liao 已提交
2238 2239
static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) {
  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
2240 2241 2242 2243
  SQInfo* pQInfo = (SQInfo*) pRuntimeEnv->qinfo;

  //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId);

D
dapan1121 已提交
2244
  //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput);
2245
//  destroyUdfInfo(pRuntimeEnv->pUdfInfo);
H
Haojun Liao 已提交
2246
  destroyDiskbasedBuf(pRuntimeEnv->pResultBuf);
2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277
  doFreeQueryHandle(pRuntimeEnv);

  destroyTsComp(pRuntimeEnv, pQueryAttr);

  pRuntimeEnv->pTsBuf = tsBufDestroy(pRuntimeEnv->pTsBuf);

  tfree(pRuntimeEnv->keyBuf);
  tfree(pRuntimeEnv->prevRow);
  tfree(pRuntimeEnv->tagVal);

  taosHashCleanup(pRuntimeEnv->pResultRowHashTable);
  pRuntimeEnv->pResultRowHashTable = NULL;

  taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap);
  pRuntimeEnv->pTableRetrieveTsMap = NULL;

  taosHashCleanup(pRuntimeEnv->pResultRowListSet);
  pRuntimeEnv->pResultRowListSet = NULL;

  destroyOperatorInfo(pRuntimeEnv->proot);

  pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool);
  taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult);
  taosArrayDestroy(pRuntimeEnv->pResultRowArrayList);
  pRuntimeEnv->prevResult = NULL;
}

static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) {
  return pQInfo->rspContext != NULL;
}

2278
bool isTaskKilled(SExecTaskInfo *pTaskInfo) {
2279 2280
  // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived
  // abort current query execution.
2281 2282
  if (pTaskInfo->owner != 0 && ((taosGetTimestampSec() - pTaskInfo->cost.start/1000) > 10*getMaximumIdleDurationSec())
      /*(!needBuildResAfterQueryComplete(pTaskInfo))*/) {
2283

2284 2285
    assert(pTaskInfo->cost.start != 0);
//    qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64
2286
//           ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec());
H
Haojun Liao 已提交
2287
//    return true;
2288 2289 2290 2291 2292
  }

  return false;
}

H
Haojun Liao 已提交
2293
void setTaskKilled(SExecTaskInfo *pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;}
2294

H
Haojun Liao 已提交
2295
//static bool isFixedOutputQuery(STaskAttr* pQueryAttr) {
2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320
//  if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) {
//    return false;
//  }
//
//  // Note:top/bottom query is fixed output query
//  if (pQueryAttr->topBotQuery || pQueryAttr->groupbyColumn || pQueryAttr->tsCompQuery) {
//    return true;
//  }
//
//  for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
//    SSqlExpr *pExpr = &pQueryAttr->pExpr1[i].base;
//
//    if (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_TS_DUMMY) {
//      continue;
//    }
//
//    if (!IS_MULTIOUTPUT(aAggs[pExpr->functionId].status)) {
//      return true;
//    }
//  }
//
//  return false;
//}

// todo refactor with isLastRowQuery
H
Haojun Liao 已提交
2321
//bool isPointInterpoQuery(STaskAttr *pQueryAttr) {
2322 2323 2324 2325 2326 2327 2328 2329 2330 2331
//  for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
//    int32_t functionId = pQueryAttr->pExpr1[i].base.functionId;
//    if (functionId == FUNCTION_INTERP) {
//      return true;
//    }
//  }
//
//  return false;
//}

H
Haojun Liao 已提交
2332
static bool isFirstLastRowQuery(STaskAttr *pQueryAttr) {
2333 2334 2335 2336 2337 2338 2339 2340 2341 2342
  for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
    int32_t functionID = getExprFunctionId(&pQueryAttr->pExpr1[i]);
    if (functionID == FUNCTION_LAST_ROW) {
      return true;
    }
  }

  return false;
}

H
Haojun Liao 已提交
2343
static bool isCachedLastQuery(STaskAttr *pQueryAttr) {
2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372
  for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
    int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]);
    if (functionId == FUNCTION_LAST || functionId == FUNCTION_LAST_DST) {
      continue;
    }

    return false;
  }

  if (pQueryAttr->order.order != TSDB_ORDER_DESC || !TSWINDOW_IS_EQUAL(pQueryAttr->window, TSWINDOW_DESC_INITIALIZER)) {
    return false;
  }

  if (pQueryAttr->groupbyColumn) {
    return false;
  }

  if (pQueryAttr->interval.interval > 0) {
    return false;
  }

  if (pQueryAttr->numOfFilterCols > 0 || pQueryAttr->havingNum > 0) {
    return false;
  }

  return true;
}

/////////////////////////////////////////////////////////////////////////////////////////////
H
Haojun Liao 已提交
2373 2374 2375 2376
//todo refactor : return window
void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) {
  assert(key >= keyFirst && key <= keyLast && pInterval->sliding <= pInterval->interval);
  win->skey = taosTimeTruncate(key, pInterval, precision);
2377 2378

  /*
H
Haojun Liao 已提交
2379
   * if the realSkey > INT64_MAX - pInterval->interval, the query duration between
2380 2381
   * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges.
   */
H
Haojun Liao 已提交
2382 2383
  if (keyFirst > (INT64_MAX - pInterval->interval)) {
    assert(keyLast - keyFirst < pInterval->interval);
2384
    win->ekey = INT64_MAX;
H
Haojun Liao 已提交
2385 2386
  } else if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
    win->ekey = taosTimeAdd(win->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
2387
  } else {
H
Haojun Liao 已提交
2388
    win->ekey = win->skey + pInterval->interval - 1;
2389 2390 2391 2392 2393 2394
  }
}

/*
 * todo add more parameters to check soon..
 */
H
Haojun Liao 已提交
2395
bool colIdCheck(STaskAttr *pQueryAttr, uint64_t qId) {
2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408
  // load data column information is incorrect
  for (int32_t i = 0; i < pQueryAttr->numOfCols - 1; ++i) {
    if (pQueryAttr->tableCols[i].colId == pQueryAttr->tableCols[i + 1].colId) {
      //qError("QInfo:0x%"PRIx64" invalid data load column for query", qId);
      return false;
    }
  }

  return true;
}

// todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which
// the scan order is not matter
H
Haojun Liao 已提交
2409
static bool onlyOneQueryType(STaskAttr *pQueryAttr, int32_t functId, int32_t functIdDst) {
2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425
  for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
    int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]);

    if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG ||
        functionId == FUNCTION_TAG_DUMMY) {
      continue;
    }

    if (functionId != functId && functionId != functIdDst) {
      return false;
    }
  }

  return true;
}

H
Haojun Liao 已提交
2426
static bool onlyFirstQuery(STaskAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_FIRST, FUNCTION_FIRST_DST); }
2427

H
Haojun Liao 已提交
2428
static bool onlyLastQuery(STaskAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_LAST, FUNCTION_LAST_DST); }
2429

H
Haojun Liao 已提交
2430
static bool notContainSessionOrStateWindow(STaskAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); }
2431

H
Haojun Liao 已提交
2432
static int32_t updateBlockLoadStatus(STaskAttr *pQuery, int32_t status) {
2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465
  bool hasFirstLastFunc = false;
  bool hasOtherFunc = false;

  if (status == BLK_DATA_ALL_NEEDED || status == BLK_DATA_DISCARD) {
    return status;
  }

  for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
    int32_t functionId = getExprFunctionId(&pQuery->pExpr1[i]);

    if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG ||
        functionId == FUNCTION_TAG_DUMMY) {
      continue;
    }

    if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) {
      hasFirstLastFunc = true;
    } else {
      hasOtherFunc = true;
    }
  }

  if (hasFirstLastFunc && status == BLK_DATA_NO_NEEDED) {
    if(!hasOtherFunc) {
      return BLK_DATA_DISCARD;
    } else {
      return BLK_DATA_ALL_NEEDED;
    }
  }

  return status;
}

H
Haojun Liao 已提交
2466
static void doUpdateLastKey(STaskAttr* pQueryAttr) {
2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484
  STimeWindow* win = &pQueryAttr->window;

  size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList);
  for(int32_t i = 0; i < num; ++i) {
    SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i);

    size_t len = taosArrayGetSize(p1);
    for(int32_t j = 0; j < len; ++j) {
//      STableKeyInfo* pInfo = taosArrayGet(p1, j);
//
//      // update the new lastkey if it is equalled to the value of the old skey
//      if (pInfo->lastKey == win->ekey) {
//        pInfo->lastKey = win->skey;
//      }
    }
  }
}

S
Shengliang Guan 已提交
2485
static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool stableQuery) {
H
Haojun Liao 已提交
2486
  STaskAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr;
2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498

  // in case of point-interpolation query, use asc order scan
  char msg[] = "QInfo:0x%"PRIx64" scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64
               "-%" PRId64 ", new qrange:%" PRId64 "-%" PRId64;

  // todo handle the case the the order irrelevant query type mixed up with order critical query type
  // descending order query for last_row query
  if (isFirstLastRowQuery(pQueryAttr)) {
    //qDebug("QInfo:0x%"PRIx64" scan order changed for last_row query, old:%d, new:%d", pQInfo->qId, pQueryAttr->order.order, TSDB_ORDER_ASC);

    pQueryAttr->order.order = TSDB_ORDER_ASC;
    if (pQueryAttr->window.skey > pQueryAttr->window.ekey) {
dengyihao's avatar
dengyihao 已提交
2499
      TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
2500 2501 2502 2503 2504 2505 2506 2507 2508
    }

    pQueryAttr->needReverseScan = false;
    return;
  }

  if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) {
    pQueryAttr->order.order = TSDB_ORDER_ASC;
    if (pQueryAttr->window.skey > pQueryAttr->window.ekey) {
dengyihao's avatar
dengyihao 已提交
2509
      TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
2510 2511 2512 2513 2514 2515 2516 2517 2518 2519
    }

    pQueryAttr->needReverseScan = false;
    doUpdateLastKey(pQueryAttr);
    return;
  }

  if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) {
    if (!QUERY_IS_ASC_QUERY(pQueryAttr)) {
      //qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
dengyihao's avatar
dengyihao 已提交
2520
      TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532
    }

    pQueryAttr->order.order = TSDB_ORDER_ASC;
    return;
  }

  if (pQueryAttr->interval.interval == 0) {
    if (onlyFirstQuery(pQueryAttr)) {
      if (!QUERY_IS_ASC_QUERY(pQueryAttr)) {
        //qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey,
//               pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);

dengyihao's avatar
dengyihao 已提交
2533
        TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
2534 2535 2536 2537 2538 2539 2540 2541 2542 2543
        doUpdateLastKey(pQueryAttr);
      }

      pQueryAttr->order.order = TSDB_ORDER_ASC;
      pQueryAttr->needReverseScan = false;
    } else if (onlyLastQuery(pQueryAttr) && notContainSessionOrStateWindow(pQueryAttr)) {
      if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
        //qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey,
//               pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);

dengyihao's avatar
dengyihao 已提交
2544
        TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558
        doUpdateLastKey(pQueryAttr);
      }

      pQueryAttr->order.order = TSDB_ORDER_DESC;
      pQueryAttr->needReverseScan = false;
    }

  } else {  // interval query
    if (stableQuery) {
      if (onlyFirstQuery(pQueryAttr)) {
        if (!QUERY_IS_ASC_QUERY(pQueryAttr)) {
          //qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC,
//                 pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);

dengyihao's avatar
dengyihao 已提交
2559
          TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
2560 2561 2562 2563 2564 2565 2566 2567 2568 2569
          doUpdateLastKey(pQueryAttr);
        }

        pQueryAttr->order.order = TSDB_ORDER_ASC;
        pQueryAttr->needReverseScan = false;
      } else if (onlyLastQuery(pQueryAttr)) {
        if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
          //qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC,
//                 pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);

dengyihao's avatar
dengyihao 已提交
2570
          TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
2571 2572 2573 2574 2575 2576 2577 2578 2579 2580
          doUpdateLastKey(pQueryAttr);
        }

        pQueryAttr->order.order = TSDB_ORDER_DESC;
        pQueryAttr->needReverseScan = false;
      }
    }
  }
}

H
Haojun Liao 已提交
2581 2582
static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) {
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596
  int32_t MIN_ROWS_PER_PAGE = 4;

  *rowsize = (int32_t)(pQueryAttr->resultRowSize * getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery));
  int32_t overhead = sizeof(SFilePage);

  // one page contains at least two rows
  *ps = DEFAULT_INTERN_BUF_PAGE_SIZE;
  while(((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) {
    *ps = ((*ps) << 1u);
  }
}

#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR)

H
Haojun Liao 已提交
2597
//static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SqlFunctionCtx *pCtx, int32_t numOfRows) {
H
Haojun Liao 已提交
2598
//  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
2599 2600 2601 2602 2603 2604 2605 2606
//
//  if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) {
//    return true;
//  }
//
//  return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows);
//}

H
Haojun Liao 已提交
2607
static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) {
2608 2609
  STimeWindow w = {0};

dengyihao's avatar
dengyihao 已提交
2610 2611
  TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
  TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
2612 2613

  if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
H
Haojun Liao 已提交
2614
//    getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w);
2615 2616 2617 2618 2619 2620 2621
    assert(w.ekey >= pBlockInfo->window.skey);

    if (w.ekey < pBlockInfo->window.ekey) {
      return true;
    }

    while(1) {
H
Haojun Liao 已提交
2622
//      getNextTimeWindow(pQueryAttr, &w);
2623 2624 2625 2626 2627 2628 2629 2630 2631 2632
      if (w.skey > pBlockInfo->window.ekey) {
        break;
      }

      assert(w.ekey > pBlockInfo->window.ekey);
      if (w.skey <= pBlockInfo->window.ekey && w.skey > pBlockInfo->window.skey) {
        return true;
      }
    }
  } else {
H
Haojun Liao 已提交
2633
//    getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w);
2634 2635 2636 2637 2638 2639 2640
    assert(w.skey <= pBlockInfo->window.ekey);

    if (w.skey > pBlockInfo->window.skey) {
      return true;
    }

    while(1) {
H
Haojun Liao 已提交
2641
//      getNextTimeWindow(pQueryAttr, &w);
2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655
      if (w.ekey < pBlockInfo->window.skey) {
        break;
      }

      assert(w.skey < pBlockInfo->window.skey);
      if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) {
        return true;
      }
    }
  }

  return false;
}

H
Haojun Liao 已提交
2656
static int32_t doTSJoinFilter(STaskRuntimeEnv *pRuntimeEnv, TSKEY key, bool ascQuery) {
2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781
  STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf);

#if defined(_DEBUG_VIEW)
  printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n",
         elem.ts, key, elem.tag.i, pQueryAttr->order.order, pRuntimeEnv->pTsBuf->tsOrder,
         pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex);
#endif

  if (ascQuery) {
    if (key < elem.ts) {
      return TS_JOIN_TS_NOT_EQUALS;
    } else if (key > elem.ts) {
      longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN);
    }
  } else {
    if (key > elem.ts) {
      return TS_JOIN_TS_NOT_EQUALS;
    } else if (key < elem.ts) {
      longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN);
    }
  }

  return TS_JOIN_TS_EQUAL;
}

bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p) {
  bool all = true;

  for (int32_t i = 0; i < numOfRows; ++i) {
    bool qualified = false;

    for (int32_t k = 0; k < numOfFilterCols; ++k) {
      char* pElem = (char*)pFilterInfo[k].pData + pFilterInfo[k].info.bytes * i;

      qualified = false;
      for (int32_t j = 0; j < pFilterInfo[k].numOfFilters; ++j) {
        SColumnFilterElem* pFilterElem = NULL;
//        SColumnFilterElem* pFilterElem = &pFilterInfo[k].pFilters[j];

        bool isnull = isNull(pElem, pFilterInfo[k].info.type);
        if (isnull) {
//          if (pFilterElem->fp == isNullOperator) {
//            qualified = true;
//            break;
//          } else {
//            continue;
//          }
        } else {
//          if (pFilterElem->fp == notNullOperator) {
//            qualified = true;
//            break;
//          } else if (pFilterElem->fp == isNullOperator) {
//            continue;
//          }
        }

        if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo[k].info.type)) {
          qualified = true;
          break;
        }
      }

      if (!qualified) {
        break;
      }
    }

    p[i] = qualified ? 1 : 0;
    if (!qualified) {
      all = false;
    }
  }

  return all;
}

void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) {
  int32_t len = 0;
  int32_t start = 0;
  for (int32_t j = 0; j < numOfRows; ++j) {
    if (p[j] == 1) {
      len++;
    } else {
      if (len > 0) {
        int32_t cstart = j - len;
        for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
          SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i);

          int16_t bytes = pColumnInfoData->info.bytes;
          memmove(((char*)pColumnInfoData->pData) + start * bytes, pColumnInfoData->pData + cstart * bytes,
                  len * bytes);
        }

        start += len;
        len = 0;
      }
    }
  }

  if (len > 0) {
    int32_t cstart = numOfRows - len;
    for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
      SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i);

      int16_t bytes = pColumnInfoData->info.bytes;
      memmove(pColumnInfoData->pData + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes);
    }

    start += len;
    len = 0;
  }

  pBlock->info.rows = start;
  pBlock->pBlockAgg = NULL;  // clean the block statistics info

  if (start > 0) {
    SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, 0);
    if (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP &&
        pColumnInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
      pBlock->info.window.skey = *(int64_t*)pColumnInfoData->pData;
      pBlock->info.window.ekey = *(int64_t*)(pColumnInfoData->pData + TSDB_KEYSIZE * (start - 1));
    }
  }
}

H
Haojun Liao 已提交
2782
void filterRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols,
2783 2784 2785 2786 2787
                           SSDataBlock* pBlock, bool ascQuery) {
  int32_t numOfRows = pBlock->info.rows;

  int8_t *p = calloc(numOfRows, sizeof(int8_t));
  bool    all = true;
H
Haojun Liao 已提交
2788
#if 0
2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815
  if (pRuntimeEnv->pTsBuf != NULL) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);

    TSKEY* k = (TSKEY*) pColInfoData->pData;
    for (int32_t i = 0; i < numOfRows; ++i) {
      int32_t offset = ascQuery? i:(numOfRows - i - 1);
      int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery);
      if (ret == TS_JOIN_TAG_NOT_EQUALS) {
        break;
      } else if (ret == TS_JOIN_TS_NOT_EQUALS) {
        all = false;
        continue;
      } else {
        assert(ret == TS_JOIN_TS_EQUAL);
        p[offset] = true;
      }

      if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) {
        break;
      }
    }

    // save the cursor status
    pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf);
  } else {
    all = doFilterDataBlock(pFilterInfo, numOfFilterCols, numOfRows, p);
  }
H
Haojun Liao 已提交
2816
#endif
2817 2818 2819 2820 2821 2822 2823 2824

  if (!all) {
    doCompactSDataBlock(pBlock, numOfRows, p);
  }

  tfree(p);
}

H
Haojun Liao 已提交
2825
void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) {
2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854
 int32_t numOfRows = pBlock->info.rows;

 int8_t *p = NULL;
 bool    all = true;

 if (pRuntimeEnv->pTsBuf != NULL) {
   SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);   
   p = calloc(numOfRows, sizeof(int8_t));
   
   TSKEY* k = (TSKEY*) pColInfoData->pData;
   for (int32_t i = 0; i < numOfRows; ++i) {
     int32_t offset = ascQuery? i:(numOfRows - i - 1);
     int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery);
     if (ret == TS_JOIN_TAG_NOT_EQUALS) {
       break;
     } else if (ret == TS_JOIN_TS_NOT_EQUALS) {
       all = false;
       continue;
     } else {
       assert(ret == TS_JOIN_TS_EQUAL);
       p[offset] = true;
     }

     if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) {
       break;
     }
   }

   // save the cursor status
H
Haojun Liao 已提交
2855
//   pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf);
2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875
 } else {
//   all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, &p, pBlock->pBlockAgg, pRuntimeEnv->pQueryAttr->numOfCols);
 }

 if (!all) {
   if (p) {
     doCompactSDataBlock(pBlock, numOfRows, p);
   } else {
     pBlock->info.rows = 0;
     pBlock->pBlockAgg = NULL;  // clean the block statistics info
   }
 }

 tfree(p);
}

static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId);
static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes);

static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) {
H
Haojun Liao 已提交
2876
  SqlFunctionCtx* pCtx = pTableScanInfo->pCtx;
2877 2878 2879 2880 2881
  uint32_t status = BLK_DATA_NO_NEEDED;

  int32_t numOfOutput = pTableScanInfo->numOfOutput;
  for (int32_t i = 0; i < numOfOutput; ++i) {
    int32_t functionId = pCtx[i].functionId;
2882
    int32_t colId = pTableScanInfo->pExpr[i].base.pColumns->info.colId;
2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912

    // group by + first/last should not apply the first/last block filter
    if (functionId < 0) {
      status |= BLK_DATA_ALL_NEEDED;
      return status;
    } else {
//      status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId);
//      if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) {
//        return status;
//      }
    }
  }

  return status;
}

void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock) {
  // set the initial static data value filter expression
  for (int32_t i = 0; i < numOfFilterCols; ++i) {
    for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) {
      SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j);

      if (pFilterInfo[i].info.colId == pColInfo->info.colId) {
        pFilterInfo[i].pData = pColInfo->pData;
        break;
      }
    }
  }
}

2913
int32_t loadDataBlock(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) {
H
Haojun Liao 已提交
2914 2915 2916 2917 2918 2919 2920 2921
  STaskCostInfo* pCost = &pTaskInfo->cost;

  pCost->totalBlocks += 1;
  pCost->totalRows += pBlock->info.rows;

  pCost->totalCheckedRows += pBlock->info.rows;
  pCost->loadBlocks += 1;

2922
  pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
H
Haojun Liao 已提交
2923 2924
  if (pBlock->pDataBlock == NULL) {
    return terrno;
2925 2926
  } else {
    return TSDB_CODE_SUCCESS;
H
Haojun Liao 已提交
2927 2928 2929
  }
}

2930
int32_t loadDataBlockOnDemand(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) {
2931 2932
  *status = BLK_DATA_NO_NEEDED;

H
Haojun Liao 已提交
2933 2934 2935 2936 2937
  pBlock->pDataBlock = NULL;
  pBlock->pBlockAgg  = NULL;

//  int64_t groupId = pRuntimeEnv->current->groupIndex;
//  bool    ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr);
2938

H
Haojun Liao 已提交
2939
  STaskCostInfo* pCost = &pTaskInfo->cost;
2940 2941 2942

  pCost->totalBlocks += 1;
  pCost->totalRows += pBlock->info.rows;
H
Haojun Liao 已提交
2943
#if 0
2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967
  if (pRuntimeEnv->pTsBuf != NULL) {
    (*status) = BLK_DATA_ALL_NEEDED;

    if (pQueryAttr->stableQuery) {  // todo refactor
      SExprInfo*   pExprInfo = &pTableScanInfo->pExpr[0];
      int16_t      tagId = (int16_t)pExprInfo->base.param[0].i;
      SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagId);

      // compare tag first
      SVariant t = {0};
      doSetTagValueInParam(pRuntimeEnv->current->pTable, tagId, &t, pColInfo->type, pColInfo->bytes);
      setTimestampListJoinInfo(pRuntimeEnv, &t, pRuntimeEnv->current);

      STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf);
      if (!tsBufIsValidElem(&elem) || (tsBufIsValidElem(&elem) && (taosVariantCompare(&t, elem.tag) != 0))) {
        (*status) = BLK_DATA_DISCARD;
        return TSDB_CODE_SUCCESS;
      }
    }
  }

  // Calculate all time windows that are overlapping or contain current data block.
  // If current data block is contained by all possible time window, do not load current data block.
  if (/*pQueryAttr->pFilters || */pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 ||
H
Haojun Liao 已提交
2968
      (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pTaskInfo, &pBlock->info))) {
2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980
    (*status) = BLK_DATA_ALL_NEEDED;
  }

  // check if this data block is required to load
  if ((*status) != BLK_DATA_ALL_NEEDED) {
    bool needFilter = true;

    // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet,
    // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer
    if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) {
      SResultRow* pResult = NULL;

H
Haojun Liao 已提交
2981
      bool  masterScan = IS_MAIN_SCAN(pRuntimeEnv);
2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009
      TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey;

      STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr);
      if (pQueryAttr->pointInterpQuery) {
        needFilter = chkWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId,
                                    pTableScanInfo->pCtx, pTableScanInfo->numOfOutput,
                                    pTableScanInfo->rowCellInfoOffset);
      } else {
        if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.uid, &win, masterScan, &pResult, groupId,
                                    pTableScanInfo->pCtx, pTableScanInfo->numOfOutput,
                                    pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) {
          longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
        }
      }
    } else if (pQueryAttr->stableQuery && (!pQueryAttr->tsCompQuery) && (!pQueryAttr->diffQuery)) { // stable aggregate, not interval aggregate or normal column aggregate
      doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx,
                               pTableScanInfo->rowCellInfoOffset, pTableScanInfo->numOfOutput,
                               pRuntimeEnv->current->groupIndex);
    }

    if (needFilter) {
      (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock);
    } else {
      (*status) = BLK_DATA_ALL_NEEDED;
    }
  }

  SDataBlockInfo* pBlockInfo = &pBlock->info;
H
Haojun Liao 已提交
3010
//  *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status);
3011 3012 3013 3014 3015 3016 3017 3018

  if ((*status) == BLK_DATA_NO_NEEDED || (*status) == BLK_DATA_DISCARD) {
    //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey,
//           pBlockInfo->window.ekey, pBlockInfo->rows);
    pCost->discardBlocks += 1;
  } else if ((*status) == BLK_DATA_STATIS_NEEDED) {
    // this function never returns error?
    pCost->loadBlockStatis += 1;
3019
//    tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg);
3020 3021

    if (pBlock->pBlockAgg == NULL) {  // data block statistics does not exist, load data block
3022
//      pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
3023 3024 3025 3026 3027 3028 3029
      pCost->totalCheckedRows += pBlock->info.rows;
    }
  } else {
    assert((*status) == BLK_DATA_ALL_NEEDED);

    // load the data block statistics to perform further filter
    pCost->loadBlockStatis += 1;
3030
//    tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg);
3031 3032 3033 3034 3035 3036

    if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) {
      { // set previous window
        if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) {
          SResultRow* pResult = NULL;

H
Haojun Liao 已提交
3037
          bool  masterScan = IS_MAIN_SCAN(pRuntimeEnv);
3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075
          TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey;

          STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr);
          if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.uid, &win, masterScan, &pResult, groupId,
                                      pTableScanInfo->pCtx, pTableScanInfo->numOfOutput,
                                      pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) {
            longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
          }
        }
      }
      bool load = false;
      for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
        int32_t functionId = pTableScanInfo->pCtx[i].functionId;
        if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) {
//          load = topbot_datablock_filter(&pTableScanInfo->pCtx[i], (char*)&(pBlock->pBlockAgg[i].min),
//                                         (char*)&(pBlock->pBlockAgg[i].max));
          if (!load) { // current block has been discard due to filter applied
            pCost->discardBlocks += 1;
            //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId,
//                   pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
            (*status) = BLK_DATA_DISCARD;
            return TSDB_CODE_SUCCESS;
          }
        }
      }
    }

    // current block has been discard due to filter applied
//    if (!doFilterByBlockStatistics(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) {
//      pCost->discardBlocks += 1;
//      qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey,
//             pBlockInfo->window.ekey, pBlockInfo->rows);
//      (*status) = BLK_DATA_DISCARD;
//      return TSDB_CODE_SUCCESS;
//    }

    pCost->totalCheckedRows += pBlockInfo->rows;
    pCost->loadBlocks += 1;
3076
//    pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088
//    if (pBlock->pDataBlock == NULL) {
//      return terrno;
//    }

//    if (pQueryAttr->pFilters != NULL) {
//      filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock);
//    }
    
//    if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) {
//      filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery);
//    }
  }
H
Haojun Liao 已提交
3089
#endif
3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156
  return TSDB_CODE_SUCCESS;
}

int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) {
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY * keyList = (TSKEY *)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
      if (key >= keyList[lastPos]) return lastPos;
      if (key == keyList[firstPos]) return firstPos;
      if (key < keyList[firstPos]) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

/*
H
Haojun Liao 已提交
3157
 * set tag value in SqlFunctionCtx
3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197
 * e.g.,tag information into input buffer
 */
static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes) {
  taosVariantDestroy(tag);

  char* val = NULL;
//  if (tagColId == TSDB_TBNAME_COLUMN_INDEX) {
//    val = tsdbGetTableName(pTable);
//    assert(val != NULL);
//  } else {
//    val = tsdbGetTableTagVal(pTable, tagColId, type, bytes);
//  }

  if (val == NULL || isNull(val, type)) {
    tag->nType = TSDB_DATA_TYPE_NULL;
    return;
  }

  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
    int32_t maxLen = bytes - VARSTR_HEADER_SIZE;
    int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val);
    taosVariantCreateFromBinary(tag, varDataVal(val), len, type);
    //taosVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type);
  } else {
    taosVariantCreateFromBinary(tag, val, bytes, type);
  }
}

static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) {
  assert(pTagColList != NULL && numOfTags > 0);

  for(int32_t i = 0; i < numOfTags; ++i) {
    if (pTagColList[i].colId == colId) {
      return &pTagColList[i];
    }
  }

  return NULL;
}

H
Haojun Liao 已提交
3198
void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
H
Haojun Liao 已提交
3199
  STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv;
3200 3201

  SExprInfo  *pExpr      = pOperatorInfo->pExpr;
H
Haojun Liao 已提交
3202
  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223

  SExprInfo* pExprInfo = &pExpr[0];
  int32_t functionId = getExprFunctionId(pExprInfo);

  if (pQueryAttr->numOfOutput == 1 && functionId == FUNCTION_TS_COMP && pQueryAttr->stableQuery) {
    assert(pExprInfo->base.numOfParams == 1);

    int16_t      tagColId = (int16_t)pExprInfo->base.param[0].i;
    SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId);

    doSetTagValueInParam(pTable, tagColId, &pCtx[0].tag, pColInfo->type, pColInfo->bytes);
    return;
  } else {
    // set tag value, by which the results are aggregated.
    int32_t offset = 0;
    memset(pRuntimeEnv->tagVal, 0, pQueryAttr->tagLen);

    for (int32_t idx = 0; idx < numOfOutput; ++idx) {
      SExprInfo* pLocalExprInfo = &pExpr[idx];

      // ts_comp column required the tag value for join filter
3224
      if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.pColumns->flag)) {
3225 3226 3227 3228
        continue;
      }

      // todo use tag column index to optimize performance
3229
      doSetTagValueInParam(pTable, pLocalExprInfo->base.pColumns->info.colId, &pCtx[idx].tag, pLocalExprInfo->base.resSchema.type,
3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256
                           pLocalExprInfo->base.resSchema.bytes);

      if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resSchema.type)
          || pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_BOOL
          || pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_TIMESTAMP) {
        memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i, pLocalExprInfo->base.resSchema.bytes);
      } else {
        if (pCtx[idx].tag.pz != NULL) {
          memcpy(pRuntimeEnv->tagVal + offset, pCtx[idx].tag.pz, pCtx[idx].tag.nLen);
        }      
      }

      offset += pLocalExprInfo->base.resSchema.bytes;
    }

    //todo : use index to avoid iterator all possible output columns
    if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) {
      setParamForStableStddev(pRuntimeEnv, pCtx, numOfOutput, pExprInfo);
    }
  }

  // set the tsBuf start position before check each data block
  if (pRuntimeEnv->pTsBuf != NULL) {
    setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable);
  }
}

H
Haojun Liao 已提交
3257
void copyToSDataBlock(SSDataBlock* pBlock, int32_t* offset, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pResBuf) {
3258 3259 3260 3261 3262 3263 3264
  pBlock->info.rows = 0;

  int32_t code = TSDB_CODE_SUCCESS;
  while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) {
    // all results in current group have been returned to client, try next group
    if ((pGroupResInfo->pRows == NULL) || taosArrayGetSize(pGroupResInfo->pRows) == 0) {
      assert(pGroupResInfo->index == 0);
H
Haojun Liao 已提交
3265
//      if ((code = mergeIntoGroupResult(&pGroupResInfo, pRuntimeEnv, offset)) != TSDB_CODE_SUCCESS) {
3266
        return;
H
Haojun Liao 已提交
3267
//      }
3268 3269
    }

H
Haojun Liao 已提交
3270
//    doCopyToSDataBlock(pResBuf, pGroupResInfo, TSDB_ORDER_ASC, pBlock, );
3271 3272 3273 3274 3275 3276 3277 3278 3279 3280

    // current data are all dumped to result buffer, clear it
    if (!hasRemainDataInCurrentGroup(pGroupResInfo)) {
      cleanupGroupResInfo(pGroupResInfo);
      if (!incNextGroup(pGroupResInfo)) {
        break;
      }
    }

    // enough results in data buffer, return
H
Haojun Liao 已提交
3281 3282 3283
//    if (pBlock->info.rows >= threshold) {
//      break;
//    }
3284 3285 3286 3287 3288 3289 3290 3291
  }
}

static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) {
  if (pTableQueryInfo == NULL) {
    return;
  }

H
Haojun Liao 已提交
3292 3293
//  TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY);
//  pTableQueryInfo->lastKey = pTableQueryInfo->win.skey;
3294

H
Haojun Liao 已提交
3295 3296
//  SWITCH_ORDER(pTableQueryInfo->cur.order);
//  pTableQueryInfo->cur.vgroupIndex = -1;
3297 3298 3299 3300 3301 3302 3303 3304 3305 3306

  // set the index to be the end slot of result rows array
  SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo;
  if (pResultRowInfo->size > 0) {
    pResultRowInfo->curPos = pResultRowInfo->size - 1;
  } else {
    pResultRowInfo->curPos = -1;
  }
}

H
Haojun Liao 已提交
3307 3308
static void setupQueryRangeForReverseScan(STableScanInfo* pTableScanInfo) {
#if 0
3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326
  int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pRuntimeEnv));
  for(int32_t i = 0; i < numOfGroups; ++i) {
    SArray *group = GET_TABLEGROUP(pRuntimeEnv, i);
    SArray *tableKeyGroup = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i);

    size_t t = taosArrayGetSize(group);
    for (int32_t j = 0; j < t; ++j) {
      STableQueryInfo *pCheckInfo = taosArrayGetP(group, j);
      updateTableQueryInfoForReverseScan(pCheckInfo);

      // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide
      // the start check timestamp of tsdbQueryHandle
//      STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j);
//      pTableKeyInfo->lastKey = pCheckInfo->lastKey;
//
//      assert(pCheckInfo->pTable == pTableKeyInfo->pTable);
    }
  }
H
Haojun Liao 已提交
3327 3328
#endif

3329 3330
}

H
Haojun Liao 已提交
3331
void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351
  for (int32_t i = 0; i < numOfOutput; ++i) {
    SWITCH_ORDER(pCtx[i].order);
  }
}

int32_t initResultRow(SResultRow *pResultRow) {
  pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow));
  pResultRow->pageId    = -1;
  pResultRow->offset    = -1;
  return TSDB_CODE_SUCCESS;
}

/*
 * The start of each column SResultRowEntryInfo is denote by RowCellInfoOffset.
 * Note that in case of top/bottom query, the whole multiple rows of result is treated as only one row of results.
 * +------------+-----------------result column 1-----------+-----------------result column 2-----------+
 * + SResultRow | SResultRowEntryInfo | intermediate buffer1 | SResultRowEntryInfo | intermediate buffer 2|
 * +------------+-------------------------------------------+-------------------------------------------+
 *           offset[0]                                  offset[1]                                   offset[2]
 */
H
Haojun Liao 已提交
3352
void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) {
H
Haojun Liao 已提交
3353
  SqlFunctionCtx* pCtx           = pInfo->pCtx;
3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385
  SSDataBlock* pDataBlock        = pInfo->pRes;
  int32_t* rowCellInfoOffset     = pInfo->rowCellInfoOffset;
  SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo;

  int64_t tid = 0;
  pRuntimeEnv->keyBuf = realloc(pRuntimeEnv->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES);
  SResultRow* pRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid);

  for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
    SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i);

    /*
     * set the output buffer information and intermediate buffer
     * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc.
     */
    struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset);
    cleanupResultRowEntry(pEntry);

    pCtx[i].resultInfo   = pEntry;
    pCtx[i].pOutput      = pData->pData;
    pCtx[i].currentStage = stage;
    assert(pCtx[i].pOutput != NULL);

    // set the timestamp output buffer for top/bottom/diff query
    int32_t fid = pCtx[i].functionId;
    if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) {
      if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput;
    }
  }

  initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols);
}
3386 3387
// TODO refactor: some function move away
void setDefaultOutputBuf_rv(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, SExecTaskInfo* pTaskInfo) {
H
Haojun Liao 已提交
3388
  SqlFunctionCtx* pCtx           = pInfo->pCtx;
H
Haojun Liao 已提交
3389 3390 3391 3392 3393
  SSDataBlock* pDataBlock        = pInfo->pRes;
  int32_t* rowCellInfoOffset     = pInfo->rowCellInfoOffset;
  SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo;

  int64_t tid = 0;
H
Haojun Liao 已提交
3394
  int64_t groupId = 0;
3395
  SResultRow* pRow = doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pSup);
H
Haojun Liao 已提交
3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421

  for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
    SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i);

    /*
     * set the output buffer information and intermediate buffer
     * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc.
     */
    struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset);
    cleanupResultRowEntry(pEntry);

    pCtx[i].resultInfo   = pEntry;
    pCtx[i].pOutput      = pData->pData;
    pCtx[i].currentStage = stage;
    assert(pCtx[i].pOutput != NULL);

    // set the timestamp output buffer for top/bottom/diff query
    int32_t fid = pCtx[i].functionId;
    if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) {
      if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput;
    }
  }

  initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols);
}

3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456
void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows) {
  SSDataBlock* pDataBlock = pBInfo->pRes;

  int32_t newSize = pDataBlock->info.rows + numOfInputRows + 5; // extra output buffer
  if ((*bufCapacity) < newSize) {
    for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
      SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i);

      char* p = realloc(pColInfo->pData, newSize * pColInfo->info.bytes);
      if (p != NULL) {
        pColInfo->pData = p;

        // it starts from the tail of the previously generated results.
        pBInfo->pCtx[i].pOutput = pColInfo->pData;
        (*bufCapacity) = newSize;
      } else {
        // longjmp
      }
    }
  }


  for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
    SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i);
    pBInfo->pCtx[i].pOutput = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows;

    // set the correct pointer after the memory buffer reallocated.
    int32_t functionId = pBInfo->pCtx[i].functionId;

    if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) {
      if (i > 0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput;
    }
  }
}

H
Haojun Liao 已提交
3457
void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499
  bool    needCopyTs = false;
  int32_t tsNum = 0;
  char *src = NULL;
  for (int32_t i = 0; i < numOfOutput; i++) {
    int32_t functionId = pCtx[i].functionId;
    if (functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) {
      needCopyTs = true;
      if (i > 0  && pCtx[i-1].functionId == FUNCTION_TS_DUMMY){
        SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data
        src = pColRes->pData;
      }
    }else if(functionId == FUNCTION_TS_DUMMY) {
      tsNum++;
    }
  }

  if (!needCopyTs) return;
  if (tsNum < 2) return;
  if (src == NULL) return;

  for (int32_t i = 0; i < numOfOutput; i++) {
    int32_t functionId = pCtx[i].functionId;
    if(functionId == FUNCTION_TS_DUMMY) {
      SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i);
      memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows);
    }
  }
}

void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) {
  SSDataBlock* pDataBlock = pBInfo->pRes;

  for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
    SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i);

    int32_t functionId = pBInfo->pCtx[i].functionId;
    if (functionId < 0) {
      memset(pBInfo->pCtx[i].pOutput, 0, pColInfo->info.bytes * (*bufCapacity));
    }
  }
}

H
Haojun Liao 已提交
3500
void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) {
3501 3502 3503 3504 3505 3506
  for (int32_t j = 0; j < size; ++j) {
    struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
    if (isRowEntryInitialized(pResInfo)) {
      continue;
    }

H
Haojun Liao 已提交
3507
    pCtx[j].fpSet->init(&pCtx[j], pCtx[j].resultInfo);
3508 3509 3510
  }
}

3511 3512
void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status) {
  if (status == TASK_NOT_COMPLETED) {
H
Haojun Liao 已提交
3513
    pTaskInfo->status = status;
3514 3515
  } else {
    // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first
3516
    CLEAR_QUERY_STATUS(pTaskInfo, TASK_NOT_COMPLETED);
H
Haojun Liao 已提交
3517
    pTaskInfo->status |= status;
3518 3519 3520
  }
}

H
Haojun Liao 已提交
3521
static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
H
Haojun Liao 已提交
3522 3523 3524 3525 3526
//  if (pRuntimeEnv->pTsBuf) {
//    SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order);
//    bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf);
//    assert(ret);
//  }
3527 3528

  // reverse order time range
H
Haojun Liao 已提交
3529 3530
  SET_REVERSE_SCAN_FLAG(pTableScanInfo);
//  setTaskStatus(pTableScanInfo, QUERY_NOT_COMPLETED);
3531 3532 3533

  switchCtxOrder(pCtx, numOfOutput);

H
Haojun Liao 已提交
3534 3535
  SWITCH_ORDER(pTableScanInfo->order);
  setupQueryRangeForReverseScan(pTableScanInfo);
3536 3537 3538 3539

  pTableScanInfo->times        = 1;
  pTableScanInfo->current      = 0;
  pTableScanInfo->reverseTimes = 0;
3540 3541
}

H
Haojun Liao 已提交
3542
void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) {
3543
  int32_t numOfOutput = pOperator->numOfOutput;
H
Haojun Liao 已提交
3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575
//  if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) {
//    // for each group result, call the finalize function for each column
//    if (pQueryAttr->groupbyColumn) {
//      closeAllResultRows(pResultRowInfo);
//    }
//
//    for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
//      SResultRow *buf = pResultRowInfo->pResult[i];
//      if (!isResultRowClosed(pResultRowInfo, i)) {
//        continue;
//      }
//
//      setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset);
//
//      for (int32_t j = 0; j < numOfOutput; ++j) {
////        pCtx[j].startTs  = buf->win.skey;
////        if (pCtx[j].functionId < 0) {
////          doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
////        } else {
////          aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]);
////        }
//      }
//
//
//      /*
//       * set the number of output results for group by normal columns, the number of output rows usually is 1 except
//       * the top and bottom query
//       */
//      buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput);
//    }
//
//  } else {
3576 3577 3578 3579
    for (int32_t j = 0; j < numOfOutput; ++j) {
//      if (pCtx[j].functionId < 0) {
//        doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
//      } else {
H
Haojun Liao 已提交
3580
        pCtx[j].fpSet->finalize(&pCtx[j]);
3581 3582
//      }
    }
H
Haojun Liao 已提交
3583
//  }
3584 3585
}

H
Haojun Liao 已提交
3586
static bool hasMainOutput(STaskAttr *pQueryAttr) {
3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597
  for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
    int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]);

    if (functionId != FUNCTION_TS && functionId != FUNCTION_TAG && functionId != FUNCTION_TAGPRJ) {
      return true;
    }
  }

  return false;
}

H
Haojun Liao 已提交
3598
STableQueryInfo *createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win) {
3599 3600 3601 3602
  STableQueryInfo *pTableQueryInfo = buf;
  pTableQueryInfo->lastKey = win.skey;

  // set more initial size of interval/groupby query
H
Haojun Liao 已提交
3603
//  if (/*QUERY_IS_INTERVAL_QUERY(pQueryAttr) || */groupbyColumn) {
3604
    int32_t initialSize = 128;
3605
    int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize);
3606 3607 3608
    if (code != TSDB_CODE_SUCCESS) {
      return NULL;
    }
H
Haojun Liao 已提交
3609 3610
//  } else { // in other aggregate query, do not initialize the windowResInfo
//  }
3611 3612 3613 3614 3615 3616 3617

  return pTableQueryInfo;
}

STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) {
  STableQueryInfo* pTableQueryInfo = calloc(1, sizeof(STableQueryInfo));

H
Haojun Liao 已提交
3618
//  pTableQueryInfo->win = win;
3619 3620 3621 3622
  pTableQueryInfo->lastKey = win.skey;

  // set more initial size of interval/groupby query
  int32_t initialSize = 16;
3623
  int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize);
3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636
  if (code != TSDB_CODE_SUCCESS) {
    tfree(pTableQueryInfo);
    return NULL;
  }

  return pTableQueryInfo;
}

void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) {
  if (pTableQueryInfo == NULL) {
    return;
  }

3637
//  taosVariantDestroy(&pTableQueryInfo->tag);
3638 3639 3640
  cleanupResultRowInfo(&pTableQueryInfo->resInfo);
}

H
Haojun Liao 已提交
3641
void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx,
3642 3643
    int32_t numOfOutput, int32_t* rowCellInfoOffset) {
  // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
3644
  SFilePage* bufPage = getBufPage(pRuntimeEnv->pResultBuf, pResult->pageId);
3645 3646 3647 3648 3649 3650 3651

  int32_t offset = 0;
  for (int32_t i = 0; i < numOfOutput; ++i) {
    pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset);

    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) {
H
Haojun Liao 已提交
3652
      offset += pCtx[i].resDataInfo.bytes;
3653 3654 3655 3656
      continue;
    }

    pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, bufPage, pResult->offset, offset);
H
Haojun Liao 已提交
3657
    offset += pCtx[i].resDataInfo.bytes;
3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673

    int32_t functionId = pCtx[i].functionId;
    if (functionId < 0) {
      continue;
    }

    if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
      if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput;
    }

//    if (!pResInfo->initialized) {
//      aAggs[functionId].init(&pCtx[i], pResInfo);
//    }
  }
}

3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706
void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf * pBuf, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) {
  // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
  SFilePage* bufPage = getBufPage(pBuf, pResult->pageId);

  int32_t offset = 0;
  for (int32_t i = 0; i < numOfOutput; ++i) {
    pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset);

    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) {
      offset += pCtx[i].resDataInfo.bytes;
      continue;
    }

    pCtx[i].pOutput = getPosInResultPage_rv(bufPage, pResult->offset, offset);
    offset += pCtx[i].resDataInfo.bytes;

    int32_t functionId = pCtx[i].functionId;
    if (functionId < 0) {
      continue;
    }

    if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
      if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i - 1].pOutput;
    }

    //    if (!pResInfo->initialized) {
    //      aAggs[functionId].init(&pCtx[i], pResInfo);
    //    }
  }
}

void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId, SExecTaskInfo* pTaskInfo) {
3707 3708 3709 3710
  // for simple group by query without interval, all the tables belong to one group result.
  int64_t uid = 0;
  int64_t tid = 0;

3711 3712 3713 3714
  SResultRowInfo* pResultRowInfo = &pAggInfo->binfo.resultRowInfo;
  SqlFunctionCtx* pCtx = pAggInfo->binfo.pCtx;
  int32_t* rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset;

3715
  SResultRow* pResultRow =
3716
      doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid, pTaskInfo, false, &pAggInfo->aggSup);
3717 3718 3719 3720 3721 3722 3723
  assert (pResultRow != NULL);

  /*
   * not assign result buffer yet, add new result buffer
   * all group belong to one result set, and each group result has different group id so set the id to be one
   */
  if (pResultRow->pageId == -1) {
3724
    int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->pResultBuf, tableGroupId, pAggInfo->binfo.resRowSize);
3725 3726 3727 3728 3729
    if (ret != TSDB_CODE_SUCCESS) {
      return;
    }
  }

3730
  setResultRowOutputBufInitCtx_rv(pAggInfo->pResultBuf, pResultRow, pCtx, numOfOutput, rowCellInfoOffset);
3731 3732
}

3733
void setExecutionContext(int32_t numOfOutput, int32_t tableGroupId, TSKEY nextKey, SExecTaskInfo* pTaskInfo, STableQueryInfo *pTableQueryInfo, SAggOperatorInfo* pAggInfo) {
3734 3735
  // lastKey needs to be updated
  pTableQueryInfo->lastKey = nextKey;
3736
  if (pAggInfo->groupId != INT32_MIN && pAggInfo->groupId == tableGroupId) {
3737 3738 3739
    return;
  }

3740
  doSetTableGroupOutputBuf(pAggInfo, numOfOutput, tableGroupId, pTaskInfo);
3741 3742

  // record the current active group id
3743
  pAggInfo->groupId = tableGroupId;
3744 3745
}

H
Haojun Liao 已提交
3746
void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx,
3747 3748
    int32_t numOfCols, int32_t* rowCellInfoOffset) {
  // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
3749
  SFilePage *page = getBufPage(pRuntimeEnv->pResultBuf, pResult->pageId);
3750 3751 3752 3753

  int16_t offset = 0;
  for (int32_t i = 0; i < numOfCols; ++i) {
    pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResult->offset, offset);
H
Haojun Liao 已提交
3754
    offset += pCtx[i].resDataInfo.bytes;
3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768

    int32_t functionId = pCtx[i].functionId;
    if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) {
      if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput;
    }

    /*
     * set the output buffer information and intermediate buffer,
     * not all queries require the interResultBuf, such as COUNT
     */
    pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset);
  }
}

H
Haojun Liao 已提交
3769
void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) {
H
Haojun Liao 已提交
3770
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784

  SSqlExpr* pExpr = &pExprInfo->base;
//  if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) &&
//      (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_PRJ) &&
//      (pExpr->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_ID)) {
//    assert(pExpr->numOfParams == 1);
//
//    int16_t      tagColId = (int16_t)pExprInfo->base.param[0].i;
//    SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId);
//
//    doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes);
//
//    int16_t tagType = pCtx[0].tag.nType;
//    if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) {
3785
//      //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_TASKID(pRuntimeEnv),
3786 3787
////             pExprInfo->base.param[0].i, pCtx[0].tag.pz);
//    } else {
3788
//      //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_TASKID(pRuntimeEnv),
3789 3790 3791 3792 3793
////             pExprInfo->base.param[0].i, pCtx[0].tag.i);
//    }
//  }
}

H
Haojun Liao 已提交
3794 3795
int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo) {
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
3796 3797

  assert(pRuntimeEnv->pTsBuf != NULL);
H
Haojun Liao 已提交
3798
#if 0
3799 3800 3801 3802 3803 3804 3805 3806 3807
  // both the master and supplement scan needs to set the correct ts comp start position
  if (pTableQueryInfo->cur.vgroupIndex == -1) {
    taosVariantAssign(&pTableQueryInfo->tag, pTag);

    STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQueryAttr->vgId, &pTableQueryInfo->tag);

    // failed to find data with the specified tag value and vnodeId
    if (!tsBufIsValidElem(&elem)) {
      if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
3808
        //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_TASKID(pRuntimeEnv), pTag->pz);
3809
      } else {
3810
        //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_TASKID(pRuntimeEnv), pTag->i);
3811 3812 3813 3814 3815 3816 3817 3818
      }

      return -1;
    }

    // Keep the cursor info of current table
    pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf);
    if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
3819
      //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
3820
    } else {
3821
      //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
3822 3823 3824 3825 3826
    }

  } else {
    tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur);
    if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
3827
      //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
3828
    } else {
3829
      //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
3830 3831
    }
  }
H
Haojun Liao 已提交
3832
#endif
3833 3834 3835 3836
  return 0;
}

// TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function.
H
Haojun Liao 已提交
3837
void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) {
3838
#if 0
H
Haojun Liao 已提交
3839
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860

  int32_t numOfExprs = pQueryAttr->numOfOutput;
  for(int32_t i = 0; i < numOfExprs; ++i) {
    SExprInfo* pExprInfo1 = &(pExprInfo[i]);
    if (pExprInfo1->base.functionId != FUNCTION_STDDEV_DST) {
      continue;
    }

    SSqlExpr* pExpr = &pExprInfo1->base;

    pCtx[i].param[0].arr = NULL;
    pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT;  // avoid freeing the memory by setting the type to be int

    // TODO use hash to speedup this loop
    int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult);
    for (int32_t j = 0; j < numOfGroup; ++j) {
      SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j);
      if (pQueryAttr->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQueryAttr->tagLen) == 0) {
        int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult);
        for (int32_t k = 0; k < numOfCols; ++k) {
          SStddevInterResult* pres = taosArrayGet(p->pResult, k);
3861
          if (pres->info.colId == pExpr->colInfo.colId) {
3862 3863 3864 3865 3866 3867 3868 3869 3870 3871
            pCtx[i].param[0].arr = pres->pResult;
            break;
          }
        }
      }
    }
  }
#endif
}

H
Haojun Liao 已提交
3872
void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) {
H
Haojun Liao 已提交
3873
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892
#if 0
  int32_t numOfExprs = pQueryAttr->numOfOutput;
  for(int32_t i = 0; i < numOfExprs; ++i) {
    SSqlExpr* pExpr1 = &pExpr[i].base;
    if (pExpr1->functionId != FUNCTION_STDDEV_DST) {
      continue;
    }

    pCtx[i].param[0].arr = NULL;
    pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT;  // avoid freeing the memory by setting the type to be int

    // TODO use hash to speedup this loop
    int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult);
    for (int32_t j = 0; j < numOfGroup; ++j) {
      SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j);
      if (bytes == 0 || memcmp(p->tags, val, bytes) == 0) {
        int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult);
        for (int32_t k = 0; k < numOfCols; ++k) {
          SStddevInterResult* pres = taosArrayGet(p->pResult, k);
3893
          if (pres->info.colId == pExpr1->colInfo.colId) {
3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912
            pCtx[i].param[0].arr = pres->pResult;
            break;
          }
        }
      }
    }
  }
#endif
}

/*
 * There are two cases to handle:
 *
 * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including pQueryAttr->lastKey,
 *    pQueryAttr->window.skey, and pQueryAttr->eKey.
 * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be
 *    merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there
 *    is a previous result generated or not.
 */
H
Haojun Liao 已提交
3913 3914
void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) {
  STaskAttr           *pQueryAttr = pRuntimeEnv->pQueryAttr;
3915 3916 3917 3918 3919 3920 3921
  STableQueryInfo  *pTableQueryInfo = pRuntimeEnv->current;
  SResultRowInfo   *pResultRowInfo = &pTableQueryInfo->resInfo;

  if (pResultRowInfo->curPos != -1) {
    return;
  }

H
Haojun Liao 已提交
3922
//  pTableQueryInfo->win.skey = key;
3923 3924 3925 3926 3927 3928 3929 3930 3931 3932
  STimeWindow win = {.skey = key, .ekey = pQueryAttr->window.ekey};

  /**
   * In handling the both ascending and descending order super table query, we need to find the first qualified
   * timestamp of this table, and then set the first qualified start timestamp.
   * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional
   * operations involve.
   */
  STimeWindow w = TSWINDOW_INITIALIZER;

dengyihao's avatar
dengyihao 已提交
3933 3934
  TSKEY sk = TMIN(win.skey, win.ekey);
  TSKEY ek = TMAX(win.skey, win.ekey);
H
Haojun Liao 已提交
3935
//  getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w);
3936 3937 3938 3939 3940 3941 3942 3943 3944

//  if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) {
//    if (!QUERY_IS_ASC_QUERY(pQueryAttr)) {
//      assert(win.ekey == pQueryAttr->window.ekey);
//    }
//
//    pResultRowInfo->prevSKey = w.skey;
//  }

H
Haojun Liao 已提交
3945
//  pTableQueryInfo->lastKey = pTableQueryInfo->win.skey;
3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957
}

/**
 * copyToOutputBuf support copy data in ascending/descending order
 * For interval query of both super table and table, copy the data in ascending order, since the output results are
 * ordered in SWindowResutl already. While handling the group by query for both table and super table,
 * all group result are completed already.
 *
 * @param pQInfo
 * @param result
 */

H
Haojun Liao 已提交
3958
static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity) {
3959 3960 3961 3962 3963 3964
  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
  int32_t numOfResult = pBlock->info.rows; // there are already exists result rows

  int32_t start = 0;
  int32_t step = -1;

3965
  //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_TASKID(pRuntimeEnv));
3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983
  assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC);

  if (orderType == TSDB_ORDER_ASC) {
    start = pGroupResInfo->index;
    step = 1;
  } else {  // desc order copy all data
    start = numOfRows - pGroupResInfo->index - 1;
    step = -1;
  }

  for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) {
    SResultRow* pRow = taosArrayGetP(pGroupResInfo->pRows, i);
    if (pRow->numOfRows == 0) {
      pGroupResInfo->index += 1;
      continue;
    }

    int32_t numOfRowsToCopy = pRow->numOfRows;
H
Haojun Liao 已提交
3984
    if (numOfResult + numOfRowsToCopy  >= rowCapacity) {
3985 3986 3987 3988 3989
      break;
    }

    pGroupResInfo->index += 1;

H
Haojun Liao 已提交
3990
    SFilePage *page = getBufPage(pBuf, pRow->pageId);
3991 3992 3993 3994 3995 3996 3997

    int32_t offset = 0;
    for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) {
      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j);
      int32_t bytes = pColInfoData->info.bytes;

      char *out = pColInfoData->pData + numOfResult * bytes;
H
Haojun Liao 已提交
3998
      char *in  = getPosInResultPage_rv(page, pRow->offset, offset);
3999 4000 4001 4002 4003 4004
      memcpy(out, in, bytes * numOfRowsToCopy);

      offset += bytes;
    }

    numOfResult += numOfRowsToCopy;
H
Haojun Liao 已提交
4005
    if (numOfResult == rowCapacity) {  // output buffer is full
4006 4007 4008 4009
      break;
    }
  }

4010
  //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_TASKID(pRuntimeEnv));
4011 4012 4013 4014
  pBlock->info.rows = numOfResult;
  return 0;
}

H
Haojun Liao 已提交
4015
static void toSDatablock(SGroupResInfo *pGroupResInfo, SDiskbasedBuf* pBuf, SSDataBlock* pBlock, int32_t rowCapacity) {
4016 4017 4018 4019 4020 4021 4022
  assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup);

  pBlock->info.rows = 0;
  if (!hasRemainDataInCurrentGroup(pGroupResInfo)) {
    return;
  }

H
Haojun Liao 已提交
4023
  int32_t orderType = TSDB_ORDER_ASC;//(pQueryAttr->pGroupbyExpr != NULL) ? pQueryAttr->pGroupbyExpr->orderType : TSDB_ORDER_ASC;
H
Haojun Liao 已提交
4024
  doCopyToSDataBlock(pBuf, pGroupResInfo, orderType, pBlock, rowCapacity);
4025

H
Haojun Liao 已提交
4026 4027
  // add condition (pBlock->info.rows >= 1) just to runtime happy
  blockDataUpdateTsWindow(pBlock);
4028 4029
}

4030
static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutput,
4031 4032
                                        SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) {
  // update the number of result for each, only update the number of rows for the corresponding window result.
4033 4034 4035
//  if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) {
//    return;
//  }
4036 4037 4038 4039 4040 4041 4042 4043 4044 4045

  for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
    SResultRow *pResult = pResultRowInfo->pResult[i];

    for (int32_t j = 0; j < numOfOutput; ++j) {
      int32_t functionId = pCtx[j].functionId;
      if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ) {
        continue;
      }

4046 4047
      SResultRowEntryInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset);
      pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes));
4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058
    }
  }
}

static int32_t compressQueryColData(SColumnInfoData *pColRes, int32_t numOfRows, char *data, int8_t compressed) {
  int32_t colSize = pColRes->info.bytes * numOfRows;
  return (*(tDataTypes[pColRes->info.type].compFunc))(pColRes->pData, colSize, numOfRows, data,
                                                                 colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0);
}

static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data, int8_t compressed, int32_t *compLen) {
H
Haojun Liao 已提交
4059 4060
  STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127

  SSDataBlock* pRes = pRuntimeEnv->outputBuf;

  int32_t *compSizes = NULL;
  int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput;

  if (compressed) {
    compSizes = calloc(numOfCols, sizeof(int32_t));
  }

  if (pQueryAttr->pExpr2 == NULL) {
    for (int32_t col = 0; col < numOfCols; ++col) {
      SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col);
      if (compressed) {
        compSizes[col] = compressQueryColData(pColRes, pRes->info.rows, data, compressed);
        data += compSizes[col];
        *compLen += compSizes[col];
        compSizes[col] = htonl(compSizes[col]);
      } else {
        memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows);
        data += pColRes->info.bytes * pRes->info.rows;
      }
    }
  } else {
    for (int32_t col = 0; col < numOfCols; ++col) {
      SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col);
      if (compressed) {
        compSizes[col] = htonl(compressQueryColData(pColRes, numOfRows, data, compressed));
        data += compSizes[col];
        *compLen += compSizes[col];
        compSizes[col] = htonl(compSizes[col]);
      } else {
        memmove(data, pColRes->pData, pColRes->info.bytes * numOfRows);
        data += pColRes->info.bytes * numOfRows;
      }
    }
  }

  if (compressed) {
    memmove(data, (char *)compSizes, numOfCols * sizeof(int32_t));
    data += numOfCols * sizeof(int32_t);

    tfree(compSizes);
  }

  int32_t numOfTables = (int32_t) taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap);
  *(int32_t*)data = htonl(numOfTables);
  data += sizeof(int32_t);

  int32_t total = 0;
  STableIdInfo* item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, NULL);

  while(item) {
    STableIdInfo* pDst = (STableIdInfo*)data;
    pDst->uid = htobe64(item->uid);
    pDst->key = htobe64(item->key);

    data += sizeof(STableIdInfo);
    total++;

    //qDebug("QInfo:0x%"PRIx64" set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo->qId, item->tid, item->uid, item->key);
    item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, item);
  }

  //qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total);

  // Check if query is completed or not for stable query or normal table query respectively.
4128
  if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) {
H
Haojun Liao 已提交
4129
//    setTaskStatus(pOperator->pTaskInfo, QUERY_OVER);
4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159
  }
}

int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock *pOutput, int32_t capacity, void** p) {
//  for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
//    SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i);
//    p[i] = pColInfoData->pData + (pColInfoData->info.bytes * pOutput->info.rows);
//  }

  int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, p, capacity - pOutput->info.rows);
  pOutput->info.rows += numOfRows;

  return pOutput->info.rows;
}

void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType) {
  SQueryProfEvent event = {0};

  event.eventType    = eventType;
  event.eventTime    = taosGetTimestampUs();
  event.operatorType = operatorInfo->operatorType;

  if (operatorInfo->pRuntimeEnv) {
    SQInfo* pQInfo = operatorInfo->pRuntimeEnv->qinfo;
    if (pQInfo->summary.queryProfEvents) {
      taosArrayPush(pQInfo->summary.queryProfEvents, &event);
    }
  }
}

4160
void publishQueryAbortEvent(SExecTaskInfo * pTaskInfo, int32_t code) {
4161 4162 4163 4164 4165
  SQueryProfEvent event;
  event.eventType = QUERY_PROF_QUERY_ABORT;
  event.eventTime = taosGetTimestampUs();
  event.abortCode = code;

4166 4167
  if (pTaskInfo->cost.queryProfEvents) {
    taosArrayPush(pTaskInfo->cost.queryProfEvents, &event);
4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244
  }
}

typedef struct  {
  uint8_t operatorType;
  int64_t beginTime;
  int64_t endTime;
  int64_t selfTime;
  int64_t descendantsTime;
} SOperatorStackItem;

static void doOperatorExecProfOnce(SOperatorStackItem* item, SQueryProfEvent* event, SArray* opStack, SHashObj* profResults) {
  item->endTime = event->eventTime;
  item->selfTime = (item->endTime - item->beginTime) - (item->descendantsTime);

  for (int32_t j = 0; j < taosArrayGetSize(opStack); ++j) {
    SOperatorStackItem* ancestor = taosArrayGet(opStack, j);
    ancestor->descendantsTime += item->selfTime;
  }

  uint8_t operatorType = item->operatorType;
  SOperatorProfResult* result = taosHashGet(profResults, &operatorType, sizeof(operatorType));
  if (result != NULL) {
    result->sumRunTimes++;
    result->sumSelfTime += item->selfTime;
  } else {
    SOperatorProfResult opResult;
    opResult.operatorType = operatorType;
    opResult.sumSelfTime = item->selfTime;
    opResult.sumRunTimes = 1;
    taosHashPut(profResults, &(operatorType), sizeof(operatorType),
                &opResult, sizeof(opResult));
  }
}

void calculateOperatorProfResults(SQInfo* pQInfo) {
  if (pQInfo->summary.queryProfEvents == NULL) {
    //qDebug("QInfo:0x%"PRIx64" query prof events array is null", pQInfo->qId);
    return;
  }

  if (pQInfo->summary.operatorProfResults == NULL) {
    //qDebug("QInfo:0x%"PRIx64" operator prof results hash is null", pQInfo->qId);
    return;
  }

  SArray* opStack = taosArrayInit(32, sizeof(SOperatorStackItem));
  if (opStack == NULL) {
    return;
  }

  size_t size = taosArrayGetSize(pQInfo->summary.queryProfEvents);
  SHashObj* profResults = pQInfo->summary.operatorProfResults;

  for (int i = 0; i < size; ++i) {
    SQueryProfEvent* event = taosArrayGet(pQInfo->summary.queryProfEvents, i);
    if (event->eventType == QUERY_PROF_BEFORE_OPERATOR_EXEC) {
      SOperatorStackItem opItem;
      opItem.operatorType = event->operatorType;
      opItem.beginTime = event->eventTime;
      opItem.descendantsTime = 0;
      taosArrayPush(opStack, &opItem);
    } else if (event->eventType == QUERY_PROF_AFTER_OPERATOR_EXEC) {
      SOperatorStackItem* item = taosArrayPop(opStack);
      assert(item->operatorType == event->operatorType);
      doOperatorExecProfOnce(item, event, opStack, profResults);
    } else if (event->eventType == QUERY_PROF_QUERY_ABORT) {
      SOperatorStackItem* item;
      while ((item = taosArrayPop(opStack)) != NULL) {
        doOperatorExecProfOnce(item, event, opStack, profResults);
      }
    }
  }

  taosArrayDestroy(opStack);
}

H
Haojun Liao 已提交
4245 4246
void queryCostStatis(SExecTaskInfo *pTaskInfo) {
  STaskCostInfo *pSummary = &pTaskInfo->cost;
4247

H
Haojun Liao 已提交
4248 4249 4250
//  uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable);
//  hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map);
//  pSummary->hashSize = hashSize;
4251 4252 4253 4254

  // add the merge time
  pSummary->elapsedTime += pSummary->firstStageMergeTime;

H
Haojun Liao 已提交
4255 4256 4257 4258 4259 4260 4261 4262 4263 4264
//  SResultRowPool* p = pTaskInfo->pool;
//  if (p != NULL) {
//    pSummary->winInfoSize = getResultRowPoolMemSize(p);
//    pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p);
//  } else {
//    pSummary->winInfoSize = 0;
//    pSummary->numOfTimeWindows = 0;
//  }
//
//  calculateOperatorProfResults(pQInfo);
4265

H
Haojun Liao 已提交
4266
  qDebug("%s :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, "
H
Haojun Liao 已提交
4267
         "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64,
H
Haojun Liao 已提交
4268
         GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis,
H
Haojun Liao 已提交
4269 4270
         pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows);
//
4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283
  //qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0,
//      pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0);

  if (pSummary->operatorProfResults) {
    SOperatorProfResult* opRes = taosHashIterate(pSummary->operatorProfResults, NULL);
    while (opRes != NULL) {
      //qDebug("QInfo:0x%" PRIx64 " :cost summary: operator : %d, exec times: %" PRId64 ", self time: %" PRId64,
//             pQInfo->qId, opRes->operatorType, opRes->sumRunTimes, opRes->sumSelfTime);
      opRes = taosHashIterate(pSummary->operatorProfResults, opRes);
    }
  }
}

H
Haojun Liao 已提交
4284 4285
//static void updateOffsetVal(STaskRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) {
//  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303
//  STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
//
//  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
//
//  if (pQueryAttr->limit.offset == pBlockInfo->rows) {  // current block will ignore completed
//    pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQueryAttr) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step;
//    pQueryAttr->limit.offset = 0;
//    return;
//  }
//
//  if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
//    pQueryAttr->pos = (int32_t)pQueryAttr->limit.offset;
//  } else {
//    pQueryAttr->pos = pBlockInfo->rows - (int32_t)pQueryAttr->limit.offset - 1;
//  }
//
//  assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1);
//
4304
//  SArray *         pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315
//  SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
//
//  // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value
//  TSKEY *keys = (TSKEY *) pColInfoData->pData;
//
//  // update the offset value
//  pTableQueryInfo->lastKey = keys[pQueryAttr->pos];
//  pQueryAttr->limit.offset = 0;
//
//  int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock);
//
4316
//  //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_TASKID(pRuntimeEnv),
4317 4318 4319
//         pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey);
//}

H
Haojun Liao 已提交
4320 4321
//void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) {
//  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
4322 4323 4324 4325 4326 4327 4328 4329 4330
//
//  if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0) {
//    return;
//  }
//
//  pQueryAttr->pos = 0;
//  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
//
//  STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
4331
//  TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle;
4332 4333
//
//  SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
4334 4335
//  while (tsdbNextDataBlock(pTsdbReadHandle)) {
//    if (isTaskKilled(pRuntimeEnv->qinfo)) {
4336 4337 4338
//      longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
//    }
//
4339
//    tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo);
4340 4341 4342 4343 4344 4345
//
//    if (pQueryAttr->limit.offset > blockInfo.rows) {
//      pQueryAttr->limit.offset -= blockInfo.rows;
//      pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey;
//      pTableQueryInfo->lastKey += step;
//
4346
//      //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_TASKID(pRuntimeEnv), blockInfo.rows,
4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358
//             pQuery->limit.offset);
//    } else {  // find the appropriated start position in current block
//      updateOffsetVal(pRuntimeEnv, &blockInfo);
//      break;
//    }
//  }
//
//  if (terrno != TSDB_CODE_SUCCESS) {
//    longjmp(pRuntimeEnv->env, terrno);
//  }
//}

H
Haojun Liao 已提交
4359 4360
//static TSKEY doSkipIntervalProcess(STaskRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) {
//  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371
//  SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo;
//
//  assert(pQueryAttr->limit.offset == 0);
//  STimeWindow tw = *win;
//  getNextTimeWindow(pQueryAttr, &tw);
//
//  if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) ||
//      (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) {
//
//    // load the data block and check data remaining in current data block
//    // TODO optimize performance
4372
//    SArray *         pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394
//    SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
//
//    tw = *win;
//    int32_t startPos =
//        getNextQualifiedWindow(pQueryAttr, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1);
//    assert(startPos >= 0);
//
//    // set the abort info
//    pQueryAttr->pos = startPos;
//
//    // reset the query start timestamp
//    pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos];
//    pQueryAttr->window.skey = pTableQueryInfo->win.skey;
//    TSKEY key = pTableQueryInfo->win.skey;
//
//    pWindowResInfo->prevSKey = tw.skey;
//    int32_t index = pRuntimeEnv->resultRowInfo.curIndex;
//
//    int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock);
//    pRuntimeEnv->resultRowInfo.curIndex = index;  // restore the window index
//
//    //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64,
4395
//           GET_TASKID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes,
4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409
//           pQueryAttr->current->lastKey);
//
//    return key;
//  } else {  // do nothing
//    pQueryAttr->window.skey      = tw.skey;
//    pWindowResInfo->prevSKey = tw.skey;
//    pTableQueryInfo->lastKey = tw.skey;
//
//    return tw.skey;
//  }
//
//  return true;
//}

H
Haojun Liao 已提交
4410 4411
//static bool skipTimeInterval(STaskRuntimeEnv *pRuntimeEnv, TSKEY* start) {
//  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436
//  if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
//    assert(*start <= pRuntimeEnv->current->lastKey);
//  } else {
//    assert(*start >= pRuntimeEnv->current->lastKey);
//  }
//
//  // if queried with value filter, do NOT forward query start position
//  if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) {
//    return true;
//  }
//
//  /*
//   * 1. for interval without interpolation query we forward pQueryAttr->interval.interval at a time for
//   *    pQueryAttr->limit.offset times. Since hole exists, pQueryAttr->interval.interval*pQueryAttr->limit.offset value is
//   *    not valid. otherwise, we only forward pQueryAttr->limit.offset number of points
//   */
//  assert(pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL);
//
//  STimeWindow w = TSWINDOW_INITIALIZER;
//  bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr);
//
//  SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo;
//  STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current;
//
//  SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
4437 4438
//  while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) {
//    tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo);
4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483
//
//    if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
//      if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
//        getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.skey, blockInfo.window.skey, pQueryAttr->window.ekey, &w);
//        pWindowResInfo->prevSKey = w.skey;
//      }
//    } else {
//      getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.ekey, pQueryAttr->window.ekey, blockInfo.window.ekey, &w);
//      pWindowResInfo->prevSKey = w.skey;
//    }
//
//    // the first time window
//    STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQueryAttr);
//
//    while (pQueryAttr->limit.offset > 0) {
//      STimeWindow tw = win;
//
//      if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) {
//        pQueryAttr->limit.offset -= 1;
//        pWindowResInfo->prevSKey = win.skey;
//
//        // current time window is aligned with blockInfo.window.ekey
//        // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL;
//        if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) {
//          pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
//        }
//      }
//
//      if (pQueryAttr->limit.offset == 0) {
//        *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo);
//        return true;
//      }
//
//      // current window does not ended in current data block, try next data block
//      getNextTimeWindow(pQueryAttr, &tw);
//
//      /*
//       * If the next time window still starts from current data block,
//       * load the primary timestamp column first, and then find the start position for the next queried time window.
//       * Note that only the primary timestamp column is required.
//       * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required
//       * time window resides in current data block.
//       */
//      if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) {
//
4484
//        SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519
//        SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
//
//        if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) {
//          pQueryAttr->limit.offset -= 1;
//        }
//
//        if (pQueryAttr->limit.offset == 0) {
//          *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo);
//          return true;
//        } else {
//          tw = win;
//          int32_t startPos =
//              getNextQualifiedWindow(pQueryAttr, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
//          assert(startPos >= 0);
//
//          // set the abort info
//          pQueryAttr->pos = startPos;
//          pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos];
//          pWindowResInfo->prevSKey = tw.skey;
//          win = tw;
//        }
//      } else {
//        break;  // offset is not 0, and next time window begins or ends in the next block.
//      }
//    }
//  }
//
//  // check for error
//  if (terrno != TSDB_CODE_SUCCESS) {
//    longjmp(pRuntimeEnv->env, terrno);
//  }
//
//  return true;
//}

4520
int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num) {
H
Haojun Liao 已提交
4521
  if (p->pDownstream == NULL) {
H
Haojun Liao 已提交
4522
    assert(p->numOfDownstream == 0);
4523 4524
  }

4525 4526 4527 4528 4529 4530 4531 4532
  p->pDownstream = calloc(1, num * POINTER_BYTES);
  if (p->pDownstream == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  memcpy(p->pDownstream, pDownstream, num * POINTER_BYTES);
  p->numOfDownstream = num;
  return TSDB_CODE_SUCCESS;
4533 4534 4535 4536
}

static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo);

H
Haojun Liao 已提交
4537 4538
static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) {
  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563
#if 0
  // TODO set the tags scan handle
  if (onlyQueryTags(pQueryAttr)) {
    return TSDB_CODE_SUCCESS;
  }

  STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window);
  if (pQueryAttr->tsCompQuery || pQueryAttr->pointInterpQuery) {
    cond.type = BLOCK_LOAD_TABLE_SEQ_ORDER;
  }

  if (!isSTableQuery
    && (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 1)
    && (cond.order == TSDB_ORDER_ASC)
    && (!QUERY_IS_INTERVAL_QUERY(pQueryAttr))
    && (!pQueryAttr->groupbyColumn)
    && (!pQueryAttr->simpleAgg)
  ) {
    SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0);
    STableQueryInfo* pCheckInfo = taosArrayGetP(pa, 0);
    cond.twindow = pCheckInfo->win;
  }

  terrno = TSDB_CODE_SUCCESS;
  if (isFirstLastRowQuery(pQueryAttr)) {
4564
    pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584

    // update the query time window
    pQueryAttr->window = cond.twindow;
    if (pQueryAttr->tableGroupInfo.numOfTables == 0) {
      pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0;
    } else {
      size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pRuntimeEnv);
      for(int32_t i = 0; i < numOfGroups; ++i) {
        SArray *group = GET_TABLEGROUP(pRuntimeEnv, i);

        size_t t = taosArrayGetSize(group);
        for (int32_t j = 0; j < t; ++j) {
          STableQueryInfo *pCheckInfo = taosArrayGetP(group, j);

          pCheckInfo->win = pQueryAttr->window;
          pCheckInfo->lastKey = pCheckInfo->win.skey;
        }
      }
    }
  } else if (isCachedLastQuery(pQueryAttr)) {
4585
    pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
4586
  } else if (pQueryAttr->pointInterpQuery) {
4587
    pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
4588
  } else {
4589
    pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
4590 4591 4592 4593 4594 4595 4596
  }
#endif
  return terrno;
}

int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr, int32_t tbScanner, SArray* pOperator,
    void* param) {
H
Haojun Liao 已提交
4597
  STaskRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
4598

H
Haojun Liao 已提交
4599
  STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr;
4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628
  pQueryAttr->tsdb = tsdb;

  if (tsdb != NULL) {
    int32_t code = setupQueryHandle(tsdb, pRuntimeEnv, pQInfo->qId, pQueryAttr->stableQuery);
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }

  pQueryAttr->interBufSize = getOutputInterResultBufSize(pQueryAttr);

  pRuntimeEnv->groupResInfo.totalGroup = (int32_t) (pQueryAttr->stableQuery? GET_NUM_OF_TABLEGROUP(pRuntimeEnv):0);
  pRuntimeEnv->enableGroupData = false;

  pRuntimeEnv->pQueryAttr = pQueryAttr;
  pRuntimeEnv->pTsBuf = pTsBuf;
  pRuntimeEnv->cur.vgroupIndex = -1;
  setResultBufSize(pQueryAttr, &pRuntimeEnv->resultInfo);

  if (sourceOptr != NULL) {
    assert(pRuntimeEnv->proot == NULL);
    pRuntimeEnv->proot = sourceOptr;
  }

  if (pTsBuf != NULL) {
    int16_t order = (pQueryAttr->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
    tsBufSetTraverseOrder(pRuntimeEnv->pTsBuf, order);
  }

H
Haojun Liao 已提交
4629
  int32_t ps = 4096;
4630 4631 4632
  getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize);

  int32_t TENMB = 1024*1024*10;
H
Haojun Liao 已提交
4633
  int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, "/tmp");
4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657
  if (code != TSDB_CODE_SUCCESS) {
    return code;
  }

  // create runtime environment
  int32_t numOfTables = (int32_t)pQueryAttr->tableGroupInfo.numOfTables;
  pQInfo->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo));
  pQInfo->summary.queryProfEvents = taosArrayInit(512, sizeof(SQueryProfEvent));
  if (pQInfo->summary.queryProfEvents == NULL) {
    //qDebug("QInfo:0x%"PRIx64" failed to allocate query prof events array", pQInfo->qId);
  }

  pQInfo->summary.operatorProfResults =
      taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK);

  if (pQInfo->summary.operatorProfResults == NULL) {
    //qDebug("QInfo:0x%"PRIx64" failed to allocate operator prof results hash", pQInfo->qId);
  }

  code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQueryAttr->tableGroupInfo.numOfTables, pOperator, param);
  if (code != TSDB_CODE_SUCCESS) {
    return code;
  }

H
Haojun Liao 已提交
4658
//  setTaskStatus(pOperator->pTaskInfo, QUERY_NOT_COMPLETED);
4659 4660 4661
  return TSDB_CODE_SUCCESS;
}

4662
static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) {
H
Haojun Liao 已提交
4663 4664
#if 0
    if (order == TSDB_ORDER_ASC) {
4665 4666
    assert(
        (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) &&
H
Haojun Liao 已提交
4667 4668
        (pTableQueryInfo->lastKey >= pTaskInfo->window.skey) &&
        (pTableQueryInfo->win.skey >= pTaskInfo->window.skey && pTableQueryInfo->win.ekey <= pTaskInfo->window.ekey));
4669 4670 4671
  } else {
    assert(
        (pTableQueryInfo->win.skey >= pTableQueryInfo->win.ekey) &&
H
Haojun Liao 已提交
4672 4673
        (pTableQueryInfo->lastKey <= pTaskInfo->window.skey) &&
        (pTableQueryInfo->win.skey <= pTaskInfo->window.skey && pTableQueryInfo->win.ekey >= pTaskInfo->window.ekey));
4674
  }
H
Haojun Liao 已提交
4675 4676
#endif

4677 4678
}

H
Haojun Liao 已提交
4679
//STsdbQueryCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win) {
4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720
//  STsdbQueryCond cond = {
//      .colList   = pQueryAttr->tableCols,
//      .order     = pQueryAttr->order.order,
//      .numOfCols = pQueryAttr->numOfCols,
//      .type      = BLOCK_LOAD_OFFSET_SEQ_ORDER,
//      .loadExternalRows = false,
//  };
//
//  TIME_WINDOW_COPY(cond.twindow, *win);
//  return cond;
//}

static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) {
  STableIdInfo tidInfo;
//  STableId* id = TSDB_TABLEID(pTableQueryInfo->pTable);
//
//  tidInfo.uid = id->uid;
//  tidInfo.tid = id->tid;
//  tidInfo.key = pTableQueryInfo->lastKey;

  return tidInfo;
}

//static void updateTableIdInfo(STableQueryInfo* pTableQueryInfo, SSDataBlock* pBlock, SHashObj* pTableIdInfo, int32_t order) {
//  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);
//  pTableQueryInfo->lastKey = ((order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey) + step;
//
//  if (pTableQueryInfo->pTable == NULL) {
//    return;
//  }
//
//  STableIdInfo tidInfo = createTableIdInfo(pTableQueryInfo);
//  STableIdInfo *idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid));
//  if (idinfo != NULL) {
//    assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid);
//    idinfo->key = tidInfo.key;
//  } else {
//    taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo));
//  }
//}

H
Haojun Liao 已提交
4721
static void doCloseAllTimeWindow(STaskRuntimeEnv* pRuntimeEnv) {
4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737
  size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv);
  for (int32_t i = 0; i < numOfGroup; ++i) {
    SArray* group = GET_TABLEGROUP(pRuntimeEnv, i);

    size_t num = taosArrayGetSize(group);
    for (int32_t j = 0; j < num; ++j) {
      STableQueryInfo* item = taosArrayGetP(group, j);
      closeAllResultRows(&item->resInfo);
    }
  }
}

static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) {
  SOperatorInfo    *pOperator = (SOperatorInfo*) param;

  STableScanInfo   *pTableScanInfo = pOperator->info;
4738
  SExecTaskInfo    *pTaskInfo = pOperator->pTaskInfo;
H
Haojun Liao 已提交
4739

4740
  SSDataBlock      *pBlock = &pTableScanInfo->block;
H
Haojun Liao 已提交
4741
  STableGroupInfo  *pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo;
4742 4743

  *newgroup = false;
H
Haojun Liao 已提交
4744

4745 4746 4747
  while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) {
    if (isTaskKilled(pOperator->pTaskInfo)) {
      longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
4748 4749 4750
    }

    pTableScanInfo->numOfBlocks += 1;
4751
    tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info);
4752 4753

    // todo opt
H
Haojun Liao 已提交
4754 4755 4756 4757 4758 4759 4760 4761 4762 4763
//    if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) {
//      STableQueryInfo** pTableQueryInfo =
//          (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid));
//      if (pTableQueryInfo == NULL) {
//        break;
//      }
//
//      pRuntimeEnv->current = *pTableQueryInfo;
//      doTableQueryInfoTimeWindowCheck(pTaskInfo, *pTableQueryInfo, pTableScanInfo->order);
//    }
4764 4765 4766

    // this function never returns error?
    uint32_t status;
H
Haojun Liao 已提交
4767 4768
    int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status);
//    int32_t  code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status);
4769
    if (code != TSDB_CODE_SUCCESS) {
4770
      longjmp(pOperator->pTaskInfo->env, code);
4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786
    }

    // current block is ignored according to filter result by block statistics data, continue load the next block
    if (status == BLK_DATA_DISCARD || pBlock->info.rows == 0) {
      continue;
    }

    return pBlock;
  }

  return NULL;
}

static SSDataBlock* doTableScan(void* param, bool *newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;

H
Haojun Liao 已提交
4787
  STableScanInfo *pTableScanInfo = pOperator->info;
4788
  SExecTaskInfo  *pTaskInfo = pOperator->pTaskInfo;
4789

H
Haojun Liao 已提交
4790
  // The read handle is not initialized yet, since no qualified tables exists
H
Haojun Liao 已提交
4791 4792 4793 4794
  if (pTableScanInfo->pTsdbReadHandle == NULL) {
    return NULL;
  }

4795 4796 4797 4798 4799 4800 4801 4802 4803 4804
  SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo;
  *newgroup = false;

  while (pTableScanInfo->current < pTableScanInfo->times) {
    SSDataBlock* p = doTableScanImpl(pOperator, newgroup);
    if (p != NULL) {
      return p;
    }

    if (++pTableScanInfo->current >= pTableScanInfo->times) {
4805
      if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)*/) {
4806 4807 4808 4809 4810 4811 4812 4813
        return NULL;
      } else {
        break;
      }
    }

    // do prepare for the next round table scan operation
//    STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window);
4814
//    tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond);
4815

4816
    setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
H
Haojun Liao 已提交
4817
    pTableScanInfo->scanFlag = REPEAT_SCAN;
4818

H
Haojun Liao 已提交
4819 4820 4821 4822 4823
//    if (pTaskInfo->pTsBuf) {
//      bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf);
//      assert(ret);
//    }
//
4824 4825 4826 4827
    if (pResultRowInfo->size > 0) {
      pResultRowInfo->curPos = 0;
    }

H
Haojun Liao 已提交
4828
    qDebug("%s start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
4829
           GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
4830 4831 4832
  }

  SSDataBlock *p = NULL;
H
Haojun Liao 已提交
4833
  // todo refactor
4834
  if (pTableScanInfo->reverseTimes > 0) {
H
Haojun Liao 已提交
4835
    setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput);
4836
//    STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window);
4837
//    tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond);
4838

H
Haojun Liao 已提交
4839
    qDebug("%s start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
4840
           GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863

    if (pResultRowInfo->size > 0) {
      pResultRowInfo->curPos = pResultRowInfo->size - 1;
    }

    p = doTableScanImpl(pOperator, newgroup);
  }

  return p;
}

static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) {
  SOperatorInfo *pOperator = (SOperatorInfo*)param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  STableScanInfo *pTableScanInfo = pOperator->info;
  *newgroup = false;
#if 0
  STableBlockDist tableBlockDist = {0};
  tableBlockDist.numOfTables     = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables;

S
Shengliang Guan 已提交
4864 4865
  int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS;
  if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) {
4866 4867 4868 4869 4870 4871 4872
    ++numRowSteps;
  }
  tableBlockDist.dataBlockInfos  = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo));
  taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps);
  tableBlockDist.maxRows = INT_MIN;
  tableBlockDist.minRows = INT_MAX;

4873 4874
  tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist);
  tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle);
4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897

  SSDataBlock* pBlock = &pTableScanInfo->block;
  pBlock->info.rows   = 1;
  pBlock->info.numOfCols = 1;

  SBufferWriter bw = tbufInitWriter(NULL, false);
  blockDistInfoToBinary(&tableBlockDist, &bw);
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0);

  int32_t len = (int32_t) tbufTell(&bw);
  pColInfo->pData = malloc(len + sizeof(int32_t));

  *(int32_t*) pColInfo->pData = len;
  memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len);

  tbufCloseWriter(&bw);

  SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0);
  pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0);

  pOperator->status = OP_EXEC_DONE;
  return pBlock;
#endif
4898 4899
}

4900 4901 4902 4903 4904 4905 4906 4907
static SSDataBlock* doStreamBlockScan(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*)param;

  // NOTE: this operator never check if current status is done or not
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  SStreamBlockScanInfo* pInfo = pOperator->info;

  SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
H
Haojun Liao 已提交
4908 4909
  pBlockInfo->rows = 0;

4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937
  while (tqNextDataBlock(pInfo->readerHandle)) {
    pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo);
    if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
      terrno = pTaskInfo->code;
      return NULL;
    }

    if (pBlockInfo->rows == 0) {
      return NULL;
    }

    pInfo->pRes->pDataBlock = tqRetrieveDataBlock(pInfo->readerHandle);
    if (pInfo->pRes->pDataBlock == NULL) {
      // TODO add log
      pTaskInfo->code = terrno;
      return NULL;
    }

    break;
  }

  // record the scan action.
  pInfo->numOfExec++;
  pInfo->numOfRows += pBlockInfo->rows;

  return (pBlockInfo->rows == 0)? NULL:pInfo->pRes;
}

4938
int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) {
4939 4940
  SSourceDataInfo* pSourceDataInfo = (SSourceDataInfo*) param;
  pSourceDataInfo->pRsp = pMsg->pData;
4941

4942 4943 4944 4945
  SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp;
  pRsp->numOfRows = htonl(pRsp->numOfRows);
  pRsp->useconds  = htobe64(pRsp->useconds);
  pRsp->compLen   = htonl(pRsp->compLen);
H
Haojun Liao 已提交
4946

4947 4948
  pSourceDataInfo->status = DATA_READY;
  tsem_post(&pSourceDataInfo->pEx->ready);
H
Haojun Liao 已提交
4949 4950 4951 4952 4953 4954 4955 4956
}

static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
  assert(pMsgBody != NULL);
  tfree(pMsgBody->msgInfo.pData);
  tfree(pMsgBody);
}

S
Shengliang Guan 已提交
4957
void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
H
Haojun Liao 已提交
4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975
  SMsgSendInfo *pSendInfo = (SMsgSendInfo *) pMsg->ahandle;
  assert(pMsg->ahandle != NULL);

  SDataBuf buf = {.len = pMsg->contLen, .pData = NULL};

  if (pMsg->contLen > 0) {
    buf.pData = calloc(1, pMsg->contLen);
    if (buf.pData == NULL) {
      terrno = TSDB_CODE_OUT_OF_MEMORY;
      pMsg->code = TSDB_CODE_OUT_OF_MEMORY;
    } else {
      memcpy(buf.pData, pMsg->pCont, pMsg->contLen);
    }
  }

  pSendInfo->fp(pSendInfo->param, &buf, pMsg->code);
  rpcFreeCont(pMsg->pCont);
  destroySendMsgInfo(pSendInfo);
4976 4977
}

4978 4979
static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInfo *pTaskInfo, int32_t sourceIndex) {
  size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources);
4980

4981 4982 4983 4984 4985
  SResFetchReq* pMsg = calloc(1, sizeof(SResFetchReq));
  if (NULL == pMsg) {
    pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
    return pTaskInfo->code;
  }
4986

4987 4988
  SDownstreamSource *pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex);
  SSourceDataInfo   *pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex);
4989

4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004
  qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu,
         GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epset.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources);

  pMsg->header.vgId = htonl(pSource->addr.nodeId);
  pMsg->sId = htobe64(pSource->schedId);
  pMsg->taskId = htobe64(pSource->taskId);
  pMsg->queryId = htobe64(pTaskInfo->id.queryId);

  // send the fetch remote task result reques
  SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo));
  if (NULL == pMsgSendInfo) {
    tfree(pMsg);
    qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo));
    pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
    return pTaskInfo->code;
H
Haojun Liao 已提交
5005 5006
  }

5007 5008 5009 5010 5011
  pMsgSendInfo->param = pDataInfo;
  pMsgSendInfo->msgInfo.pData = pMsg;
  pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq);
  pMsgSendInfo->msgType = TDMT_VND_FETCH;
  pMsgSendInfo->fp = loadRemoteDataCallback;
5012

5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027
  int64_t transporterId = 0;
  int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epset, &transporterId, pMsgSendInfo);
  return TSDB_CODE_SUCCESS;
}

static int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SExchangeInfo *pExchangeInfo, SSourceDataInfo* pDataInfo, int32_t numOfOutput, int64_t startTs) {
  char* pData = pDataInfo->pRsp->data;
  SRetrieveTableRsp* pRsp = pDataInfo->pRsp;

  for (int32_t i = 0; i < numOfOutput; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i);

    char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pRsp->numOfRows);
    if (tmp == NULL) {
      return TSDB_CODE_QRY_OUT_OF_MEMORY;
5028
    }
5029

5030 5031
    size_t len = pRsp->numOfRows * pColInfoData->info.bytes;
    memcpy(tmp, pData, len);
5032

5033 5034 5035
    pColInfoData->pData = tmp;
    pData += len;
  }
5036

5037
  pRes->info.rows = pRsp->numOfRows;
5038

5039
  int64_t el = taosGetTimestampUs() - startTs;
5040

5041 5042 5043
  pExchangeInfo->totalRows += pRsp->numOfRows;
  pExchangeInfo->totalSize += pRsp->compLen;
  pDataInfo->totalRows += pRsp->numOfRows;
5044

5045
  pExchangeInfo->totalElapsed += el;
5046

5047 5048
  return TSDB_CODE_SUCCESS;
}
5049

5050 5051 5052
static void* setAllSourcesCompleted(SOperatorInfo *pOperator, int64_t startTs) {
  SExchangeInfo *pExchangeInfo = pOperator->info;
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
H
Haojun Liao 已提交
5053

5054 5055
  int64_t el = taosGetTimestampUs() - startTs;
  pExchangeInfo->totalElapsed += el;
H
Haojun Liao 已提交
5056

5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076
  size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources);
  qDebug("%s all %"PRIzu" sources are exhausted, total rows: %"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources,
         pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0);

  doSetOperatorCompleted(pOperator);
  return NULL;
}

static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo *pOperator, SExchangeInfo *pExchangeInfo, SExecTaskInfo *pTaskInfo) {
  int32_t code = 0;
  int64_t startTs = taosGetTimestampUs();
  size_t  totalSources = taosArrayGetSize(pExchangeInfo->pSources);

  while (1) {
    int32_t completed = 0;
    for (int32_t i = 0; i < totalSources; ++i) {
      SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i);

      if (pDataInfo->status == DATA_EXHAUSTED) {
        completed += 1;
H
Haojun Liao 已提交
5077 5078
        continue;
      }
5079

5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096
      if (pDataInfo->status != DATA_READY) {
        continue;
      }

      SRetrieveTableRsp* pRsp = pDataInfo->pRsp;
      SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, i);

      SSDataBlock* pRes = pExchangeInfo->pResult;

      if (pRsp->numOfRows == 0) {
        qDebug("%s vgId:%d, taskID:0x%" PRIx64 " index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 " try next",
               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, i + 1, pDataInfo->totalRows,
               pExchangeInfo->totalRows);
        pDataInfo->status = DATA_EXHAUSTED;
        completed += 1;
        continue;
      }
H
Haojun Liao 已提交
5097

5098 5099
      code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pExchangeInfo, pDataInfo, pOperator->numOfOutput, startTs);
      if (code != 0) {
5100 5101 5102
        goto _error;
      }

5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170
      if (pRsp->completed == 1) {
        qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64
               ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows,
               pDataInfo->totalRows, pExchangeInfo->totalRows, pExchangeInfo->totalSize, i + 1,
               totalSources);
        pDataInfo->status = DATA_EXHAUSTED;
      } else {
        qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64,
               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows,
               pExchangeInfo->totalSize);
      }

      if (pDataInfo->status != DATA_EXHAUSTED) {
        pDataInfo->status = DATA_NOT_READY;
        code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i);
        if (code != TSDB_CODE_SUCCESS) {
          goto _error;
        }
      }

      return pExchangeInfo->pResult;
    }

    if (completed == totalSources) {
      return setAllSourcesCompleted(pOperator, startTs);
    }
  }

_error:
  pTaskInfo->code = code;
  return NULL;
}

static SSDataBlock* concurrentlyLoadRemoteData(SOperatorInfo *pOperator) {
  SExchangeInfo *pExchangeInfo = pOperator->info;
  SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo;
  
  if (pOperator->status == OP_RES_TO_RETURN) {
    return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo);
  }

  size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources);
  int64_t startTs = taosGetTimestampUs();

  // Asynchronously send all fetch requests to all sources.
  for(int32_t i = 0; i < totalSources; ++i) {
    int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i);
    if (code != TSDB_CODE_SUCCESS) {
      return NULL;
    }
  }

  int64_t endTs = taosGetTimestampUs();
  qDebug("%s send all fetch request to %"PRIzu" sources completed, elapsed:%"PRId64, GET_TASKID(pTaskInfo), totalSources, endTs - startTs);

  tsem_wait(&pExchangeInfo->ready);

  pOperator->status = OP_RES_TO_RETURN;
  return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo);
}

static SSDataBlock* seqLoadRemoteData(SOperatorInfo *pOperator) {
  SExchangeInfo *pExchangeInfo = pOperator->info;
  SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo;

  size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources);
  int64_t startTs = taosGetTimestampUs();
5171

5172 5173 5174
  while(1) {
    if (pExchangeInfo->current >= totalSources) {
      return setAllSourcesCompleted(pOperator, startTs);
5175
    }
5176

5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188
    doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current);

    tsem_wait(&pExchangeInfo->ready);

    SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current);
    SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);

    SRetrieveTableRsp* pRsp = pDataInfo->pRsp;
    if (pRsp->numOfRows == 0) {
      qDebug("%s vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next",
             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1,
             pDataInfo->totalRows, pExchangeInfo->totalRows);
H
Haojun Liao 已提交
5189

5190 5191 5192 5193
      pDataInfo->status = DATA_EXHAUSTED;
      pExchangeInfo->current += 1;
      continue;
    }
H
Haojun Liao 已提交
5194

5195 5196
    SSDataBlock* pRes = pExchangeInfo->pResult;
    setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pExchangeInfo, pDataInfo, pOperator->numOfOutput, startTs);
5197 5198

    if (pRsp->completed == 1) {
H
Haojun Liao 已提交
5199
      qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64
5200 5201 5202 5203
                 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows,
             pDataInfo->totalRows, pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->current + 1,
             totalSources);
5204

5205
      pDataInfo->status = DATA_EXHAUSTED;
5206 5207
      pExchangeInfo->current += 1;
    } else {
H
Haojun Liao 已提交
5208
      qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64,
H
Haojun Liao 已提交
5209
             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows, pExchangeInfo->totalSize);
5210 5211 5212 5213
    }

    return pExchangeInfo->pResult;
  }
5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236
}

static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
  SOperatorInfo *pOperator = (SOperatorInfo*) param;

  SExchangeInfo *pExchangeInfo = pOperator->info;
  SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo;

  size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources);

  if (pOperator->status == OP_EXEC_DONE) {
    qDebug("%s all %"PRIzu" source(s) are exhausted, total rows:%"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources,
           pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0);
    return NULL;
  }

  *newgroup = false;

  if (pExchangeInfo->seqLoadData) {
    return seqLoadRemoteData(pOperator);
  } else {
    return concurrentlyLoadRemoteData(pOperator);
  }
H
Haojun Liao 已提交
5237

5238
#if 0
H
Haojun Liao 已提交
5239 5240 5241 5242 5243 5244
  _error:
  tfree(pMsg);
  tfree(pMsgSendInfo);

  terrno = pTaskInfo->code;
  return NULL;
5245
#endif
H
Haojun Liao 已提交
5246
}
5247

H
Haojun Liao 已提交
5248 5249
static SSDataBlock* createResultDataBlock(const SArray* pExprInfo);

H
Haojun Liao 已提交
5250
SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pExprInfo, SExecTaskInfo* pTaskInfo) {
5251 5252 5253 5254 5255 5256 5257 5258 5259 5260
  SExchangeInfo* pInfo    = calloc(1, sizeof(SExchangeInfo));
  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));

  if (pInfo == NULL || pOperator == NULL) {
    tfree(pInfo);
    tfree(pOperator);
    terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
    return NULL;
  }

5261 5262
  size_t numOfSources = taosArrayGetSize(pSources);

H
Haojun Liao 已提交
5263
  pInfo->pSources = taosArrayDup(pSources);
5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281
  pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo));
  if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) {
    tfree(pInfo);
    tfree(pOperator);
    taosArrayDestroy(pInfo->pSources);
    taosArrayDestroy(pInfo->pSourceDataInfo);
    terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
    return NULL;
  }

  for(int32_t i = 0; i < numOfSources; ++i) {
    SSourceDataInfo dataInfo = {0};
    dataInfo.status = DATA_NOT_READY;
    dataInfo.pEx    = pInfo;
    dataInfo.index  = i;

    taosArrayPush(pInfo->pSourceDataInfo, &dataInfo);
  }
H
Haojun Liao 已提交
5282 5283

  size_t size = taosArrayGetSize(pExprInfo);
H
Haojun Liao 已提交
5284
  pInfo->pResult = createResultDataBlock(pExprInfo);
5285 5286 5287
  pInfo->seqLoadData = true;

  tsem_init(&pInfo->ready, 0, 0);
5288 5289

  pOperator->name         = "ExchangeOperator";
X
Xiaoyu Wang 已提交
5290
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE;
5291 5292 5293
  pOperator->blockingOptr = false;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->info         = pInfo;
H
Haojun Liao 已提交
5294
  pOperator->numOfOutput  = size;
5295 5296 5297
  pOperator->exec         = doLoadRemoteData;
  pOperator->pTaskInfo    = pTaskInfo;

S
Shengliang Guan 已提交
5298
#if 1
H
Haojun Liao 已提交
5299
  { // todo refactor
H
Haojun Liao 已提交
5300 5301 5302
    SRpcInit rpcInit;
    memset(&rpcInit, 0, sizeof(rpcInit));
    rpcInit.localPort = 0;
H
Haojun Liao 已提交
5303
    rpcInit.label = "EX";
H
Haojun Liao 已提交
5304
    rpcInit.numOfThreads = 1;
S
Shengliang Guan 已提交
5305
    rpcInit.cfp = qProcessFetchRsp;
S
Shengliang Guan 已提交
5306
    rpcInit.sessions = tsMaxConnections;
H
Haojun Liao 已提交
5307 5308
    rpcInit.connType = TAOS_CONN_CLIENT;
    rpcInit.user = (char *)"root";
S
Shengliang Guan 已提交
5309
    rpcInit.idleTime = tsShellActivityTimer * 1000;
H
Haojun Liao 已提交
5310
    rpcInit.ckey = "key";
S
Shengliang Guan 已提交
5311
    rpcInit.spi = 1;
H
Haojun Liao 已提交
5312 5313 5314 5315 5316 5317 5318
    rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6";

    pInfo->pTransporter = rpcOpen(&rpcInit);
    if (pInfo->pTransporter == NULL) {
      return NULL; // todo
    }
  }
S
Shengliang 已提交
5319
#endif
5320

5321 5322 5323
  return pOperator;
}

H
Haojun Liao 已提交
5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348
SSDataBlock* createResultDataBlock(const SArray* pExprInfo) {
  SSDataBlock* pResBlock = calloc(1, sizeof(SSDataBlock));
  if (pResBlock == NULL) {
    return NULL;
  }

  size_t numOfCols = taosArrayGetSize(pExprInfo);
  pResBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));

  SArray* pResult = pResBlock->pDataBlock;
  for(int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData colInfoData = {0};
    SExprInfo* p = taosArrayGetP(pExprInfo, i);

    SSchema* pSchema = &p->base.resSchema;
    colInfoData.info.type  = pSchema->type;
    colInfoData.info.colId = pSchema->colId;
    colInfoData.info.bytes = pSchema->bytes;

    taosArrayPush(pResult, &colInfoData);
  }

  return pResBlock;
}

H
Haojun Liao 已提交
5349
SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) {
5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361
  assert(repeatTime > 0);

  STableScanInfo* pInfo    = calloc(1, sizeof(STableScanInfo));
  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    tfree(pInfo);
    tfree(pOperator);

    terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
    return NULL;
  }

H
Haojun Liao 已提交
5362
  pInfo->pTsdbReadHandle = pTsdbReadHandle;
5363 5364 5365 5366 5367 5368
  pInfo->times        = repeatTime;
  pInfo->reverseTimes = reverseTime;
  pInfo->order        = order;
  pInfo->current      = 0;
  pInfo->scanFlag     = MAIN_SCAN;

H
Haojun Liao 已提交
5369
  pOperator->name          = "TableScanOperator";
X
Xiaoyu Wang 已提交
5370
  pOperator->operatorType  = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN;
5371 5372 5373 5374 5375 5376 5377 5378 5379 5380
  pOperator->blockingOptr  = false;
  pOperator->status        = OP_IN_EXECUTING;
  pOperator->info          = pInfo;
  pOperator->numOfOutput   = numOfOutput;
  pOperator->exec          = doTableScan;
  pOperator->pTaskInfo     = pTaskInfo;

  return pOperator;
}

H
Haojun Liao 已提交
5381
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) {
5382 5383
  STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));

H
Haojun Liao 已提交
5384
  pInfo->pTsdbReadHandle     = pTsdbReadHandle;
5385 5386 5387 5388 5389 5390 5391 5392 5393
  pInfo->times            = 1;
  pInfo->reverseTimes     = 0;
  pInfo->order            = pRuntimeEnv->pQueryAttr->order.order;
  pInfo->current          = 0;
  pInfo->prevGroupId      = -1;
  pRuntimeEnv->enableGroupData = true;

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  pOperator->name         = "TableSeqScanOperator";
X
Xiaoyu Wang 已提交
5394
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN;
5395 5396 5397 5398 5399 5400 5401 5402 5403 5404
  pOperator->blockingOptr = false;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->info         = pInfo;
  pOperator->numOfOutput  = pRuntimeEnv->pQueryAttr->numOfCols;
  pOperator->pRuntimeEnv  = pRuntimeEnv;
  pOperator->exec         = doTableScanImpl;

  return pOperator;
}

H
Haojun Liao 已提交
5405
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) {
5406 5407
  STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));

H
Haojun Liao 已提交
5408
  pInfo->pTsdbReadHandle     = pTsdbReadHandle;
5409 5410 5411 5412 5413 5414 5415 5416 5417 5418
  pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));

  SColumnInfoData infoData = {{0}};
  infoData.info.type = TSDB_DATA_TYPE_BINARY;
  infoData.info.bytes = 1024;
  infoData.info.colId = 0;
  taosArrayPush(pInfo->block.pDataBlock, &infoData);

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  pOperator->name         = "TableBlockInfoScanOperator";
H
Haojun Liao 已提交
5419
//  pOperator->operatorType = OP_TableBlockInfoScan;
5420 5421 5422
  pOperator->blockingOptr = false;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->info         = pInfo;
H
Haojun Liao 已提交
5423
//  pOperator->numOfOutput  = pRuntimeEnv->pQueryAttr->numOfCols;
5424 5425 5426 5427 5428
  pOperator->exec         = doBlockInfoScan;

  return pOperator;
}

5429
SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) {
5430 5431 5432 5433 5434 5435 5436 5437 5438
  SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo));
  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    tfree(pInfo);
    tfree(pOperator);
    terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
    return NULL;
  }

5439 5440 5441
  // todo dynamic set the value of 4096
  pInfo->pRes = createOutputBuf_rv(pExprInfo, 4096);

H
Haojun Liao 已提交
5442 5443 5444 5445 5446 5447 5448
  int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprInfo);
  SArray* pColList = taosArrayInit(numOfOutput, sizeof(int32_t));
  for(int32_t i = 0; i < numOfOutput; ++i) {
    SExprInfo* pExpr = taosArrayGetP(pExprInfo, i);
    taosArrayPush(pColList, &pExpr->pExpr->pSchema[0].colId);
  }
  
L
Liu Jicong 已提交
5449 5450
  // set the extract column id to streamHandle
  tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList);
5451 5452 5453 5454 5455 5456
  int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList);
  if (code != 0) {
    tfree(pInfo);
    tfree(pOperator);
    return NULL;
  }
H
Haojun Liao 已提交
5457 5458

  pInfo->readerHandle = streamReadHandle;
H
Haojun Liao 已提交
5459

5460
  pOperator->name          = "StreamBlockScanOperator";
X
Xiaoyu Wang 已提交
5461
  pOperator->operatorType  = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
5462 5463 5464 5465 5466 5467
  pOperator->blockingOptr  = false;
  pOperator->status        = OP_IN_EXECUTING;
  pOperator->info          = pInfo;
  pOperator->numOfOutput   = numOfOutput;
  pOperator->exec          = doStreamBlockScan;
  pOperator->pTaskInfo     = pTaskInfo;
L
Liu Jicong 已提交
5468
  return pOperator;
H
Haojun Liao 已提交
5469 5470
}

5471 5472 5473 5474 5475
void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) {
  assert(pTableScanInfo != NULL && pDownstream != NULL);

  pTableScanInfo->pExpr = pDownstream->pExpr;   // TODO refactor to use colId instead of pExpr
  pTableScanInfo->numOfOutput = pDownstream->numOfOutput;
H
Haojun Liao 已提交
5476
#if 0
5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524
  if (pDownstream->operatorType == OP_Aggregate || pDownstream->operatorType == OP_MultiTableAggregate) {
    SAggOperatorInfo* pAggInfo = pDownstream->info;

    pTableScanInfo->pCtx = pAggInfo->binfo.pCtx;
    pTableScanInfo->pResultRowInfo = &pAggInfo->binfo.resultRowInfo;
    pTableScanInfo->rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset;
  } else if (pDownstream->operatorType == OP_TimeWindow || pDownstream->operatorType == OP_AllTimeWindow) {
    STableIntervalOperatorInfo *pIntervalInfo = pDownstream->info;

    pTableScanInfo->pCtx = pIntervalInfo->pCtx;
    pTableScanInfo->pResultRowInfo = &pIntervalInfo->resultRowInfo;
    pTableScanInfo->rowCellInfoOffset = pIntervalInfo->rowCellInfoOffset;

  } else if (pDownstream->operatorType == OP_Groupby) {
    SGroupbyOperatorInfo *pGroupbyInfo = pDownstream->info;

    pTableScanInfo->pCtx = pGroupbyInfo->binfo.pCtx;
    pTableScanInfo->pResultRowInfo = &pGroupbyInfo->binfo.resultRowInfo;
    pTableScanInfo->rowCellInfoOffset = pGroupbyInfo->binfo.rowCellInfoOffset;

  } else if (pDownstream->operatorType == OP_MultiTableTimeInterval || pDownstream->operatorType == OP_AllMultiTableTimeInterval) {
    STableIntervalOperatorInfo *pInfo = pDownstream->info;

    pTableScanInfo->pCtx = pInfo->pCtx;
    pTableScanInfo->pResultRowInfo = &pInfo->resultRowInfo;
    pTableScanInfo->rowCellInfoOffset = pInfo->rowCellInfoOffset;

  } else if (pDownstream->operatorType == OP_Project) {
    SProjectOperatorInfo *pInfo = pDownstream->info;

    pTableScanInfo->pCtx = pInfo->binfo.pCtx;
    pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo;
    pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset;
  } else if (pDownstream->operatorType == OP_SessionWindow) {
    SSWindowOperatorInfo* pInfo = pDownstream->info;

    pTableScanInfo->pCtx = pInfo->binfo.pCtx;
    pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo;
    pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset;
  } else if (pDownstream->operatorType == OP_StateWindow) {
    SStateWindowOperatorInfo* pInfo = pDownstream->info;

    pTableScanInfo->pCtx = pInfo->binfo.pCtx;
    pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo;
    pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset;
  } else {
    assert(0);
  }
H
Haojun Liao 已提交
5525 5526
#endif

5527 5528
}

H
Haojun Liao 已提交
5529
SArray* getOrderCheckColumns(STaskAttr* pQuery) {
5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555
  int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0: taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo);

  SArray* pOrderColumns = NULL;
  if (numOfCols > 0) {
    pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo);
  } else {
    pOrderColumns = taosArrayInit(4, sizeof(SColIndex));
  }

  if (pQuery->interval.interval > 0) {
    if (pOrderColumns == NULL) {
      pOrderColumns = taosArrayInit(1, sizeof(SColIndex));
    }

    SColIndex colIndex = {.colIndex = 0, .colId = 0, .flag = TSDB_COL_NORMAL};
    taosArrayPush(pOrderColumns, &colIndex);
  }

  {
    numOfCols = (int32_t) taosArrayGetSize(pOrderColumns);
    for(int32_t i = 0; i < numOfCols; ++i) {
      SColIndex* index = taosArrayGet(pOrderColumns, i);
      for(int32_t j = 0; j < pQuery->numOfOutput; ++j) {
        SSqlExpr* pExpr = &pQuery->pExpr1[j].base;
        int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]);

5556
        if (index->colId == pExpr->pColumns->info.colId &&
5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567
            (functionId == FUNCTION_PRJ || functionId == FUNCTION_TAG || functionId == FUNCTION_TS)) {
          index->colIndex = j;
          index->colId = pExpr->resSchema.colId;
        }
      }
    }
  }

  return pOrderColumns;
}

H
Haojun Liao 已提交
5568
SArray* getResultGroupCheckColumns(STaskAttr* pQuery) {
5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586
  int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo);

  SArray* pOrderColumns = NULL;
  if (numOfCols > 0) {
    pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo);
  } else {
    pOrderColumns = taosArrayInit(4, sizeof(SColIndex));
  }

  for(int32_t i = 0; i < numOfCols; ++i) {
    SColIndex* index = taosArrayGet(pOrderColumns, i);

    bool found = false;
    for(int32_t j = 0; j < pQuery->numOfOutput; ++j) {
      SSqlExpr* pExpr = &pQuery->pExpr1[j].base;
      int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]);

      // FUNCTION_TAG_DUMMY function needs to be ignored
5587
      if (index->colId == pExpr->pColumns->info.colId &&
5588 5589
          ((TSDB_COL_IS_TAG(pExpr->pColumns->flag) && functionId == FUNCTION_TAG) ||
           (TSDB_COL_IS_NORMAL_COL(pExpr->pColumns->flag) && functionId == FUNCTION_PRJ))) {
5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602
        index->colIndex = j;
        index->colId = pExpr->resSchema.colId;
        found = true;
        break;
      }
    }

    assert(found && index->colIndex >= 0 && index->colIndex < pQuery->numOfOutput);
  }

  return pOrderColumns;
}

5603 5604
static int32_t initAggSup(SAggSupporter* pAggSup, SArray* pExprInfo);
static void clearupAggSup(SAggSupporter* pAggSup);
5605

5606
static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) {
H
Haojun Liao 已提交
5607 5608
  SSortedMergeOperatorInfo* pInfo = (SSortedMergeOperatorInfo*) param;
  taosArrayDestroy(pInfo->orderInfo);
5609 5610 5611
  taosArrayDestroy(pInfo->groupInfo);

  if (pInfo->pSortHandle != NULL) {
H
Haojun Liao 已提交
5612
    tsortDestroySortHandle(pInfo->pSortHandle);
5613
  }
H
Haojun Liao 已提交
5614
  blockDataDestroy(pInfo->binfo.pRes);
5615 5616

  clearupAggSup(&pInfo->aggSup);
5617
}
H
Haojun Liao 已提交
5618

5619 5620 5621
static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) {
  SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param;
  taosArrayDestroy(pInfo->orderColumnList);
H
Haojun Liao 已提交
5622
  pInfo->pRes = blockDataDestroy(pInfo->pRes);
5623 5624 5625
  tfree(pInfo->prevRow);
}

X
Xiaoyu Wang 已提交
5626 5627
static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) {
  assert(dst != NULL && src != NULL);
5628

X
Xiaoyu Wang 已提交
5629
  *dst = *src;
5630

X
Xiaoyu Wang 已提交
5631 5632 5633
  dst->pExpr = exprdup(src->pExpr);
  dst->base.pColumns = calloc(src->base.numOfCols, sizeof(SColumn));
  memcpy(dst->base.pColumns, src->base.pColumns, sizeof(SColumn) * src->base.numOfCols);
5634

X
Xiaoyu Wang 已提交
5635 5636 5637
  memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param));
  for (int32_t j = 0; j < src->base.numOfParams; ++j) {
    taosVariantAssign(&dst->base.param[j], &src->base.param[j]);
5638
  }
X
Xiaoyu Wang 已提交
5639
}
5640

5641 5642
static SExprInfo* exprArrayDup(SArray* pExprList) {
  size_t numOfOutput = taosArrayGetSize(pExprList);
5643

H
Haojun Liao 已提交
5644
  SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo));
5645 5646
  for (int32_t i = 0; i < numOfOutput; ++i) {
    SExprInfo* pExpr = taosArrayGetP(pExprList, i);
H
Haojun Liao 已提交
5647
    assignExprInfo(&p[i], pExpr);
5648 5649
  }

H
Haojun Liao 已提交
5650 5651
  return p;
}
5652

H
Haojun Liao 已提交
5653
// TODO merge aggregate super table
5654
static void appendOneRowToDataBlock(SSDataBlock *pBlock, STupleHandle* pTupleHandle) {
5655 5656
  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
5657

H
Haojun Liao 已提交
5658
    bool isNull = tsortIsNullVal(pTupleHandle, i);
H
Haojun Liao 已提交
5659 5660 5661
    if (isNull) {
      colDataAppend(pColInfo, pBlock->info.rows, NULL, true);
    } else {
H
Haojun Liao 已提交
5662
      char* pData = tsortGetValue(pTupleHandle, i);
H
Haojun Liao 已提交
5663 5664
      colDataAppend(pColInfo, pBlock->info.rows, pData, false);
    }
5665 5666
  }

5667
  pBlock->info.rows += 1;
5668 5669
}

5670 5671
static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, bool hasVarCol, int32_t capacity) {
  blockDataClearup(pDataBlock, hasVarCol);
5672

H
Haojun Liao 已提交
5673
  while(1) {
H
Haojun Liao 已提交
5674
    STupleHandle* pTupleHandle = tsortNextTuple(pHandle);
5675 5676
    if (pTupleHandle == NULL) {
      break;
5677 5678
    }

5679 5680 5681
    appendOneRowToDataBlock(pDataBlock, pTupleHandle);
    if (pDataBlock->info.rows >= capacity) {
      return pDataBlock;
5682 5683 5684
    }
  }

5685
  return (pDataBlock->info.rows > 0)? pDataBlock:NULL;
5686
}
5687

5688
SSDataBlock* loadNextDataBlock(void* param) {
H
Haojun Liao 已提交
5689 5690
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  bool newgroup = false;
5691

H
Haojun Liao 已提交
5692
  return pOperator->exec(pOperator, &newgroup);
5693 5694
}

5695 5696 5697 5698 5699
static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char **buf, int32_t rowIndex) {
  size_t size = taosArrayGetSize(groupInfo);
  if (size == 0) {
    return true;
  }
5700

5701 5702
  for (int32_t i = 0; i < size; ++i) {
    int32_t* index = taosArrayGet(groupInfo, i);
5703

5704 5705
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, *index);
    bool isNull = colDataIsNull(pColInfo, rowIndex, pBlock->info.rows, NULL);
5706

5707 5708 5709
    if ((isNull && buf[i] != NULL) || (!isNull && buf[i] == NULL)) {
      return false;
    }
5710

5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723
    char* pCell = colDataGetData(pColInfo, rowIndex);
    if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
      if (varDataLen(pCell) != varDataLen(buf[i])) {
        return false;
      } else {
        if (memcmp(varDataVal(pCell), varDataVal(buf[i]), varDataLen(pCell)) != 0) {
          return false;
        }
      }
    } else {
      if (memcmp(pCell, buf[i], pColInfo->info.bytes) != 0) {
        return false;
      }
5724 5725
    }
  }
5726

5727
  return 0;
5728
}
H
Haojun Liao 已提交
5729

5730 5731 5732
static void doMergeResultImpl(SSortedMergeOperatorInfo* pInfo, SqlFunctionCtx *pCtx, int32_t numOfExpr, int32_t rowIndex) {
  for (int32_t j = 0; j < numOfExpr; ++j) { // TODO set row index
    pCtx[j].startRow = rowIndex;
5733 5734
  }

5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745
  for (int32_t j = 0; j < numOfExpr; ++j) {
    int32_t functionId = pCtx[j].functionId;
    pCtx[j].fpSet->addInput(&pCtx[j]);

//    if (functionId < 0) {
//      SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
//      doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE);
//    } else {
//      assert(!TSDB_FUNC_IS_SCALAR(functionId));
//      aAggs[functionId].mergeFunc(&pCtx[j]);
//    }
5746
  }
5747
}
5748

5749 5750 5751 5752 5753 5754
static void doFinalizeResultImpl(SqlFunctionCtx *pCtx, int32_t numOfExpr) {
  for(int32_t j = 0; j < numOfExpr; ++j) {
    int32_t functionId = pCtx[j].functionId;
    //    if (functionId == FUNC_TAG_DUMMY || functionId == FUNC_TS_DUMMY) {
    //      continue;
    //    }
5755

5756 5757 5758 5759 5760 5761 5762
    //    if (functionId < 0) {
    //      SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
    //      doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
    //    } else {
    pCtx[j].fpSet->addInput(&pCtx[j]);
  }
}
5763

5764 5765
static bool saveCurrentTuple(char** rowColData, SArray* pColumnList, SSDataBlock* pBlock, int32_t rowIndex) {
  int32_t size = (int32_t) taosArrayGetSize(pColumnList);
5766

5767 5768 5769
  for(int32_t i = 0; i < size; ++i) {
    int32_t* index = taosArrayGet(pColumnList, i);
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, *index);
H
Haojun Liao 已提交
5770

5771 5772 5773
    char* data = colDataGetData(pColInfo, rowIndex);
    memcpy(rowColData[i], data, colDataGetLength(pColInfo, rowIndex));
  }
5774

5775 5776
  return true;
}
5777

5778 5779
static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock* pBlock) {
  SSortedMergeOperatorInfo* pInfo = pOperator->info;
5780

5781 5782 5783 5784
  SqlFunctionCtx* pCtx = pInfo->binfo.pCtx;
  for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
    pCtx[i].size = 1;
  }
5785

5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797
  for(int32_t i = 0; i < pBlock->info.rows; ++i) {
    if (!pInfo->hasGroupVal) {
      ASSERT(i == 0);
      doMergeResultImpl(pInfo, pCtx, numOfExpr, i);
      pInfo->hasGroupVal = saveCurrentTuple(pInfo->groupVal, pInfo->groupInfo, pBlock, i);
    } else {
      if (needToMerge(pBlock, pInfo->groupInfo, pInfo->groupVal, i)) {
        doMergeResultImpl(pInfo, pCtx, numOfExpr, i);
      } else {
        doFinalizeResultImpl(pCtx, numOfExpr);
        int32_t numOfRows = getNumOfResult(pInfo->binfo.pCtx, pOperator->numOfOutput);
        //        setTagValueForMultipleRows(pCtx, pOperator->numOfOutput, numOfRows);
5798

5799
        // TODO check for available buffer;
H
Haojun Liao 已提交
5800

5801 5802 5803 5804 5805
        // next group info data
        pInfo->binfo.pRes->info.rows += numOfRows;
        for (int32_t j = 0; j < numOfExpr; ++j) {
          if (pCtx[j].functionId < 0) {
            continue;
5806
          }
5807 5808

          pCtx[j].fpSet->addInput(&pCtx[j]);
5809
        }
5810 5811 5812

        doMergeResultImpl(pInfo, pCtx, numOfExpr, i);
        pInfo->hasGroupVal = saveCurrentTuple(pInfo->groupVal, pInfo->groupInfo, pBlock, i);
H
Haojun Liao 已提交
5813
      }
5814 5815 5816 5817
    }
  }
}

5818 5819 5820
static SSDataBlock* doMerge(SOperatorInfo* pOperator) {
  SSortedMergeOperatorInfo* pInfo = pOperator->info;
  SSortHandle* pHandle = pInfo->pSortHandle;
5821

5822 5823 5824 5825 5826 5827 5828
  SSDataBlock* pDataBlock = createOneDataBlock(pInfo->binfo.pRes);
  blockDataEnsureCapacity(pDataBlock, pInfo->binfo.capacity);

  while(1) {

    blockDataClearup(pDataBlock, pInfo->hasVarCol);
    while (1) {
H
Haojun Liao 已提交
5829
      STupleHandle* pTupleHandle = tsortNextTuple(pHandle);
5830 5831
      if (pTupleHandle == NULL) {
        break;
5832
      }
5833

5834 5835 5836 5837 5838
      // build datablock for merge for one group
      appendOneRowToDataBlock(pDataBlock, pTupleHandle);
      if (pDataBlock->info.rows >= pInfo->binfo.capacity) {
        break;
      }
5839
    }
5840

5841 5842 5843
    if (pDataBlock->info.rows == 0) {
      break;
    }
5844

5845 5846 5847 5848 5849
    setInputDataBlock(pOperator, pInfo->binfo.pCtx, pDataBlock, TSDB_ORDER_ASC);
    //  updateOutputBuf(&pInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor, pOperator->pRuntimeEnv, true);
    doMergeImpl(pOperator, pOperator->numOfOutput, pDataBlock);
    // flush to tuple store, and after all data have been handled, return to upstream node or sink node
  }
5850

5851 5852 5853
  doFinalizeResultImpl(pInfo->binfo.pCtx, pOperator->numOfOutput);
  int32_t numOfRows = getNumOfResult(pInfo->binfo.pCtx, pOperator->numOfOutput);
  //        setTagValueForMultipleRows(pCtx, pOperator->numOfOutput, numOfRows);
5854

5855
  // TODO check for available buffer;
5856

5857 5858 5859 5860
  // next group info data
  pInfo->binfo.pRes->info.rows += numOfRows;
  return (pInfo->binfo.pRes->info.rows > 0)? pInfo->binfo.pRes:NULL;
}
5861

5862
static SSDataBlock* doSortedMerge(void* param, bool* newgroup) {
5863 5864 5865
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
5866 5867 5868
  }

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
5869
  SSortedMergeOperatorInfo* pInfo = pOperator->info;
H
Haojun Liao 已提交
5870
  if (pOperator->status == OP_RES_TO_RETURN) {
5871
    return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->hasVarCol, pInfo->binfo.capacity);
5872 5873
  }

H
Haojun Liao 已提交
5874
  SSchema* p = blockDataExtractSchema(pInfo->binfo.pRes, NULL);
5875
  int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
H
Haojun Liao 已提交
5876
  pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize,
H
Haojun Liao 已提交
5877
                                        numOfBufPage, p, pInfo->binfo.pRes->info.numOfCols, "GET_TASKID(pTaskInfo)");
5878

5879
  tfree(p);
H
Haojun Liao 已提交
5880
  tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock);
5881

5882
  for(int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
H
Haojun Liao 已提交
5883
    SGenericSource* ps = calloc(1, sizeof(SGenericSource));
H
Haojun Liao 已提交
5884
    ps->param = pOperator->pDownstream[i];
H
Haojun Liao 已提交
5885
    tsortAddSource(pInfo->pSortHandle, ps);
5886 5887
  }

H
Haojun Liao 已提交
5888
  int32_t code = tsortOpen(pInfo->pSortHandle);
5889
  if (code != TSDB_CODE_SUCCESS) {
5890
    longjmp(pTaskInfo->env, terrno);
5891 5892
  }

H
Haojun Liao 已提交
5893
  pOperator->status = OP_RES_TO_RETURN;
5894
  return doMerge(pOperator);
5895
}
5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911

static SArray* createBlockOrder(SArray* pExprInfo, SArray* pOrderVal) {
  SArray* pOrderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));

  size_t numOfOrder = taosArrayGetSize(pOrderVal);
  for (int32_t j = 0; j < numOfOrder; ++j) {
    SBlockOrderInfo orderInfo = {0};
    SOrder*         pOrder = taosArrayGet(pOrderVal, j);
    orderInfo.order = pOrder->order;

    for (int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) {
      SExprInfo* pExpr = taosArrayGet(pExprInfo, i);
      if (pExpr->base.resSchema.colId == pOrder->col.info.colId) {
        orderInfo.colIndex = i;
        break;
      }
5912
    }
5913

5914 5915
    taosArrayPush(pOrderInfo, &orderInfo);
  }
H
Haojun Liao 已提交
5916

5917
  return pOrderInfo;
H
Haojun Liao 已提交
5918 5919
}

5920 5921 5922
static int32_t initGroupCol(SArray* pExprInfo, SArray* pGroupInfo, SSortedMergeOperatorInfo* pInfo) {
  if (pGroupInfo == NULL || taosArrayGetSize(pGroupInfo) == 0) {
    return 0;
H
Haojun Liao 已提交
5923 5924
  }

5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943
  int32_t len = 0;
  SArray* plist = taosArrayInit(3, sizeof(SColumn));
  pInfo->groupInfo = taosArrayInit(3, sizeof(int32_t));

  if (plist == NULL || pInfo->groupInfo == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  size_t  numOfGroupCol = taosArrayGetSize(pInfo->groupInfo);
  for(int32_t i = 0; i < numOfGroupCol; ++i) {
    SColumn* pCol = taosArrayGet(pGroupInfo, i);
    for(int32_t j = 0; j < taosArrayGetSize(pExprInfo); ++j) {
      SExprInfo* pe = taosArrayGet(pExprInfo, j);
      if (pe->base.resSchema.colId == pCol->info.colId) {
        taosArrayPush(plist, pCol);
        taosArrayPush(pInfo->groupInfo, &j);
        len += pCol->info.bytes;
        break;
      }
H
Haojun Liao 已提交
5944 5945 5946
    }
  }

5947
  ASSERT(taosArrayGetSize(pGroupInfo) == taosArrayGetSize(plist));
H
Haojun Liao 已提交
5948

5949 5950 5951 5952 5953
  pInfo->groupVal = calloc(1, (POINTER_BYTES * numOfGroupCol + len));
  if (pInfo->groupVal == NULL) {
    taosArrayDestroy(plist);
    return TSDB_CODE_OUT_OF_MEMORY;
  }
H
Haojun Liao 已提交
5954

5955 5956 5957 5958 5959 5960 5961
  int32_t offset = 0;
  char   *start = (char*)(pInfo->groupVal + (POINTER_BYTES * numOfGroupCol));
  for(int32_t i = 0; i < numOfGroupCol; ++i) {
    pInfo->groupVal[i] = start + offset;
    SColumn* pCol = taosArrayGet(plist, i);
    offset += pCol->info.bytes;
  }
H
Haojun Liao 已提交
5962

5963
  taosArrayDestroy(plist);
H
Haojun Liao 已提交
5964

5965 5966
  return TSDB_CODE_SUCCESS;
}
H
Haojun Liao 已提交
5967

5968
SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo) {
5969 5970 5971
  SSortedMergeOperatorInfo* pInfo = calloc(1, sizeof(SSortedMergeOperatorInfo));
  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
5972
    goto _error;
5973
  }
H
Haojun Liao 已提交
5974

H
Haojun Liao 已提交
5975
  int32_t numOfOutput = taosArrayGetSize(pExprInfo);
5976 5977 5978
  pInfo->binfo.pCtx   = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize);
  pInfo->binfo.pRes   = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity);
  initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1);
H
Haojun Liao 已提交
5979

5980 5981 5982
  if (pInfo->binfo.pCtx == NULL || pInfo->binfo.pRes == NULL) {
    goto _error;
  }
H
Haojun Liao 已提交
5983

5984 5985 5986 5987
  int32_t code = initAggSup(&pInfo->aggSup, pExprInfo);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
H
Haojun Liao 已提交
5988

5989 5990 5991 5992 5993
  setDefaultOutputBuf_rv(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, pTaskInfo);
  code = initGroupCol(pExprInfo, pGroupInfo, pInfo);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
H
Haojun Liao 已提交
5994

5995 5996
//  pInfo->resultRowFactor = (int32_t)(getRowNumForMultioutput(pRuntimeEnv->pQueryAttr,
//      pRuntimeEnv->pQueryAttr->topBotQuery, false));
5997 5998
  pInfo->sortBufSize    = 1024 * 16; // 1MB
  pInfo->bufPageSize    = 1024;
H
Haojun Liao 已提交
5999
  pInfo->orderInfo      = createBlockOrder(pExprInfo, pOrderVal);
H
Haojun Liao 已提交
6000

6001
  pInfo->binfo.capacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, pInfo->bufPageSize);
H
Haojun Liao 已提交
6002

6003
  pOperator->name         = "SortedMerge";
X
Xiaoyu Wang 已提交
6004
  // pOperator->operatorType = OP_SortedMerge;
6005 6006 6007
  pOperator->blockingOptr = true;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->info         = pInfo;
H
Haojun Liao 已提交
6008
  pOperator->numOfOutput  = numOfOutput;
6009
  pOperator->pExpr        = exprArrayDup(pExprInfo);
H
Haojun Liao 已提交
6010

6011 6012 6013
  pOperator->pTaskInfo    = pTaskInfo;
  pOperator->exec         = doSortedMerge;
  pOperator->cleanupFn    = destroySortedMergeOperatorInfo;
H
Haojun Liao 已提交
6014

6015 6016 6017
  code = appendDownstream(pOperator, downstream, numOfDownstream);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
6018
  }
H
Haojun Liao 已提交
6019

6020
  return pOperator;
H
Haojun Liao 已提交
6021

6022 6023 6024
  _error:
  if (pInfo != NULL) {
    destroySortedMergeOperatorInfo(pInfo, numOfOutput);
H
Haojun Liao 已提交
6025 6026
  }

6027 6028 6029 6030
  tfree(pInfo);
  tfree(pOperator);
  terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
  return NULL;
H
Haojun Liao 已提交
6031 6032
}

6033 6034 6035 6036 6037 6038
static SSDataBlock* doSort(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

6039
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
6040
  SOrderOperatorInfo* pInfo = pOperator->info;
H
Haojun Liao 已提交
6041
  if (pOperator->status == OP_RES_TO_RETURN) {
6042
    return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->hasVarCol, pInfo->numOfRowsInRes);
H
Haojun Liao 已提交
6043 6044
  }

6045 6046
  SSchema* p = blockDataExtractSchema(pInfo->pDataBlock, NULL);
  int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
H
Haojun Liao 已提交
6047
  pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize,
6048
                                        numOfBufPage, p, pInfo->pDataBlock->info.numOfCols, "GET_TASKID(pTaskInfo)");
H
Haojun Liao 已提交
6049

6050
  tfree(p);
H
Haojun Liao 已提交
6051
  tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock);
H
Haojun Liao 已提交
6052

H
Haojun Liao 已提交
6053 6054
  SGenericSource* ps = calloc(1, sizeof(SGenericSource));
  ps->param = pOperator;
H
Haojun Liao 已提交
6055
  tsortAddSource(pInfo->pSortHandle, ps);
6056

6057
  // TODO set error code;
H
Haojun Liao 已提交
6058
  int32_t code = tsortOpen(pInfo->pSortHandle);
6059
  if (code != TSDB_CODE_SUCCESS) {
6060
    longjmp(pTaskInfo->env, terrno);
6061
  }
6062

H
Haojun Liao 已提交
6063
  pOperator->status = OP_RES_TO_RETURN;
6064
  return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->hasVarCol, pInfo->numOfRowsInRes);
6065 6066
}

6067
SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo) {
6068
  SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo));
H
Haojun Liao 已提交
6069 6070 6071 6072 6073 6074 6075
  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    tfree(pInfo);

    terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
    return NULL;
  }
6076

H
Haojun Liao 已提交
6077 6078 6079 6080
  pInfo->sortBufSize    = 1024 * 16; // 1MB
  pInfo->bufPageSize    = 1024;
  pInfo->numOfRowsInRes = 1024;

H
Haojun Liao 已提交
6081
  pInfo->pDataBlock  = createOutputBuf_rv(pExprInfo, pInfo->numOfRowsInRes);
6082
  pInfo->orderInfo  = createBlockOrder(pExprInfo, pOrderVal);
6083 6084 6085 6086 6087 6088 6089 6090 6091

  for(int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) {
    SExprInfo* pExpr = taosArrayGetP(pExprInfo, i);
    if (IS_VAR_DATA_TYPE(pExpr->base.resSchema.type)) {
      pInfo->hasVarCol = true;
      break;
    }
  }

6092
  if (pInfo->orderInfo == NULL || pInfo->pDataBlock == NULL) {
H
Haojun Liao 已提交
6093 6094 6095 6096 6097 6098 6099
    tfree(pOperator);
    destroyOrderOperatorInfo(pInfo, taosArrayGetSize(pExprInfo));
    tfree(pInfo);

    terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
    return NULL;
  }
6100

H
Haojun Liao 已提交
6101
  pOperator->name          = "Order";
X
Xiaoyu Wang 已提交
6102
  pOperator->operatorType  = QUERY_NODE_PHYSICAL_PLAN_SORT;
6103 6104 6105
  pOperator->blockingOptr  = true;
  pOperator->status        = OP_IN_EXECUTING;
  pOperator->info          = pInfo;
6106 6107

  pOperator->pTaskInfo     = pTaskInfo;
6108
  pOperator->exec          = doSort;
H
Haojun Liao 已提交
6109
  pOperator->cleanupFn     = destroyOrderOperatorInfo;
6110

6111
  int32_t code = appendDownstream(pOperator, &downstream, 1);
6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128
  return pOperator;
}

static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) {
  return pTableScanInfo->order;
}

// this is a blocking operator
static SSDataBlock* doAggregate(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SAggOperatorInfo* pAggInfo = pOperator->info;
  SOptrBasicInfo* pInfo = &pAggInfo->binfo;

H
Haojun Liao 已提交
6129
  int32_t order = TSDB_ORDER_ASC;
H
Haojun Liao 已提交
6130
  SOperatorInfo* downstream = pOperator->pDownstream[0];
6131 6132

  while(1) {
H
Haojun Liao 已提交
6133 6134 6135
    publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
    publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
6136 6137 6138 6139 6140

    if (pBlock == NULL) {
      break;
    }

H
Haojun Liao 已提交
6141
//    if (pAggInfo->current != NULL) {
H
Haojun Liao 已提交
6142
//      setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfOutput);
H
Haojun Liao 已提交
6143
//    }
6144 6145 6146

    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order);
H
Haojun Liao 已提交
6147
    doAggregateImpl(pOperator, 0, pInfo->pCtx, pBlock);
6148 6149 6150 6151 6152 6153 6154
  }

  doSetOperatorCompleted(pOperator);

  finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset);
  pInfo->pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput);

6155
  return (pInfo->pRes->info.rows != 0)? pInfo->pRes:NULL;
6156 6157
}

H
Haojun Liao 已提交
6158
static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) {
6159 6160 6161 6162 6163 6164 6165
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SAggOperatorInfo* pAggInfo = pOperator->info;
  SOptrBasicInfo* pInfo = &pAggInfo->binfo;
6166
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
6167 6168

  if (pOperator->status == OP_RES_TO_RETURN) {
H
Haojun Liao 已提交
6169
    toSDatablock(&pAggInfo->groupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity);
6170

H
Haojun Liao 已提交
6171
    if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) {
6172 6173 6174 6175 6176 6177
      pOperator->status = OP_EXEC_DONE;
    }

    return pInfo->pRes;
  }

6178
  // table scan order
H
Haojun Liao 已提交
6179
  int32_t order = TSDB_ORDER_ASC;
H
Haojun Liao 已提交
6180
  SOperatorInfo* downstream = pOperator->pDownstream[0];
6181 6182

  while(1) {
H
Haojun Liao 已提交
6183 6184 6185
    publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
    publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
6186 6187 6188 6189 6190

    if (pBlock == NULL) {
      break;
    }

H
Haojun Liao 已提交
6191
//    setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput);
H
Haojun Liao 已提交
6192
//    if (downstream->operatorType == OP_TableScan) {
H
Haojun Liao 已提交
6193
//      STableScanInfo* pScanInfo = downstream->info;
H
Haojun Liao 已提交
6194 6195
//      order = getTableScanOrder(pScanInfo);
//    }
6196 6197 6198 6199 6200

    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order);

    TSKEY key = 0;
6201
    if (order == TSDB_ORDER_ASC) {
6202 6203 6204 6205 6206 6207 6208
      key = pBlock->info.window.ekey;
      TSKEY_MAX_ADD(key, 1);
    } else {
      key = pBlock->info.window.skey;
      TSKEY_MIN_SUB(key, -1);
    }
    
6209 6210
    setExecutionContext(pOperator->numOfOutput, pAggInfo->current->groupIndex, key, pTaskInfo, pAggInfo->current, pAggInfo);
    doAggregateImpl(pOperator, 0, pInfo->pCtx, pBlock);
6211 6212 6213 6214
  }

  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pInfo->resultRowInfo);
6215
  updateNumOfRowsInResultRows(pInfo->pCtx, pOperator->numOfOutput, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset);
6216

H
Haojun Liao 已提交
6217 6218
  initGroupResInfo(&pAggInfo->groupResInfo, &pInfo->resultRowInfo);
  toSDatablock(&pAggInfo->groupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity);
6219

H
Haojun Liao 已提交
6220
  if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) {
6221 6222 6223 6224 6225 6226 6227 6228 6229 6230
    doSetOperatorCompleted(pOperator);
  }

  return pInfo->pRes;
}

static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;

  SProjectOperatorInfo* pProjectInfo = pOperator->info;
H
Haojun Liao 已提交
6231
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247
  SOptrBasicInfo *pInfo = &pProjectInfo->binfo;

  SSDataBlock* pRes = pInfo->pRes;
  int32_t order = pRuntimeEnv->pQueryAttr->order.order;

  pRes->info.rows = 0;

  if (pProjectInfo->existDataBlock) {  // TODO refactor
    STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;

    SSDataBlock* pBlock = pProjectInfo->existDataBlock;
    pProjectInfo->existDataBlock = NULL;
    *newgroup = true;

    // todo dynamic set tags
    if (pTableQueryInfo != NULL) {
H
Haojun Liao 已提交
6248
//      setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput);
6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267
    }

    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order);
    updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows);

    projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput);

    pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput);
    if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) {
      copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput);
      resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfOutput);
      return pRes;
    }
  }

  while(1) {
    bool prevVal = *newgroup;

H
Haojun Liao 已提交
6268
    // The downstream exec may change the value of the newgroup, so use a local variable instead.
H
Haojun Liao 已提交
6269 6270 6271
    publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
    publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
6272 6273 6274 6275 6276

    if (pBlock == NULL) {
      assert(*newgroup == false);

      *newgroup = prevVal;
6277
      setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297
      break;
    }

    // Return result of the previous group in the firstly.
    if (*newgroup) {
      if (pRes->info.rows > 0) {
        pProjectInfo->existDataBlock = pBlock;
        break;
      } else { // init output buffer for a new group data
//        for (int32_t j = 0; j < pOperator->numOfOutput; ++j) {
//          aAggs[pInfo->pCtx[j].functionId].xFinalize(&pInfo->pCtx[j]);
//        }
        initCtxOutputBuffer(pInfo->pCtx, pOperator->numOfOutput);
      }
    }

    STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;

    // todo dynamic set tags
    if (pTableQueryInfo != NULL) {
H
Haojun Liao 已提交
6298
//      setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput);
6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322
    }

    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order);
    updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows);

    projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput);
    pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput);
    if (pRes->info.rows >= 1000/*pRuntimeEnv->resultInfo.threshold*/) {
      break;
    }
  }
  copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput);
  resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfOutput);
  return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL;
}

static SSDataBlock* doLimit(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*)param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SLimitOperatorInfo* pInfo = pOperator->info;
H
Haojun Liao 已提交
6323
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
6324 6325 6326

  SSDataBlock* pBlock = NULL;
  while (1) {
H
Haojun Liao 已提交
6327 6328 6329
    publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
    pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
    publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374

    if (pBlock == NULL) {
      doSetOperatorCompleted(pOperator);
      return NULL;
    }

    if (pRuntimeEnv->currentOffset == 0) {
      break;
    } else if (pRuntimeEnv->currentOffset >= pBlock->info.rows) {
      pRuntimeEnv->currentOffset -= pBlock->info.rows;
    } else {
      int32_t remain = (int32_t)(pBlock->info.rows - pRuntimeEnv->currentOffset);
      pBlock->info.rows = remain;

      for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

        int16_t bytes = pColInfoData->info.bytes;
        memmove(pColInfoData->pData, pColInfoData->pData + bytes * pRuntimeEnv->currentOffset, remain * bytes);
      }

      pRuntimeEnv->currentOffset = 0;
      break;
    }
  }

  if (pInfo->total + pBlock->info.rows >= pInfo->limit) {
    pBlock->info.rows = (int32_t)(pInfo->limit - pInfo->total);
    pInfo->total = pInfo->limit;

    doSetOperatorCompleted(pOperator);
  } else {
    pInfo->total += pBlock->info.rows;
  }

  return pBlock;
}

static SSDataBlock* doFilter(void* param, bool* newgroup) {
  SOperatorInfo *pOperator = (SOperatorInfo *)param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SFilterOperatorInfo* pCondInfo = pOperator->info;
H
Haojun Liao 已提交
6375
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
6376 6377

  while (1) {
H
Haojun Liao 已提交
6378 6379 6380
    publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock *pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
    publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404

    if (pBlock == NULL) {
      break;
    }

    doSetFilterColumnInfo(pCondInfo->pFilterInfo, pCondInfo->numOfFilterCols, pBlock);
    assert(pRuntimeEnv->pTsBuf == NULL);
    filterRowsInDataBlock(pRuntimeEnv, pCondInfo->pFilterInfo, pCondInfo->numOfFilterCols, pBlock, true);

    if (pBlock->info.rows > 0) {
      return pBlock;
    }
  }

  doSetOperatorCompleted(pOperator);
  return NULL;
}

static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

H
Haojun Liao 已提交
6405
  STableIntervalOperatorInfo* pInfo = pOperator->info;
6406 6407

  if (pOperator->status == OP_RES_TO_RETURN) {
H
Haojun Liao 已提交
6408
//    toSDatablock(pAggInfo->pGroupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity);
H
Haojun Liao 已提交
6409
    if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
6410 6411 6412
      doSetOperatorCompleted(pOperator);
    }

H
Haojun Liao 已提交
6413
    return pInfo->binfo.pRes;
6414 6415
  }

H
Haojun Liao 已提交
6416 6417
//  int32_t order = pQueryAttr->order.order;
//  STimeWindow win = pQueryAttr->window;
H
Haojun Liao 已提交
6418
  SOperatorInfo* downstream = pOperator->pDownstream[0];
6419 6420

  while(1) {
H
Haojun Liao 已提交
6421 6422 6423
    publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
    publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
6424 6425 6426 6427 6428

    if (pBlock == NULL) {
      break;
    }

H
Haojun Liao 已提交
6429
//    setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput);
6430 6431

    // the pDataBlock are always the same one, no need to call this again
H
Haojun Liao 已提交
6432 6433
    setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC);
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0);
6434 6435 6436
  }

  // restore the value
H
Haojun Liao 已提交
6437 6438
//  pQueryAttr->order.order = order;
//  pQueryAttr->window = win;
6439 6440

  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
6441
  closeAllResultRows(&pInfo->binfo.resultRowInfo);
6442
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
6443
  finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset);
6444

H
Haojun Liao 已提交
6445 6446
  initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
  toSDatablock(&pInfo->groupResInfo, pInfo->pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity);
6447

H
Haojun Liao 已提交
6448
  if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
6449 6450 6451
    doSetOperatorCompleted(pOperator);
  }

H
Haojun Liao 已提交
6452
  return pInfo->binfo.pRes->info.rows == 0? NULL:pInfo->binfo.pRes;
6453 6454 6455 6456 6457 6458 6459 6460 6461 6462
}

static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  STableIntervalOperatorInfo* pIntervalInfo = pOperator->info;

H
Haojun Liao 已提交
6463
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
6464
  if (pOperator->status == OP_RES_TO_RETURN) {
H
Haojun Liao 已提交
6465
//    toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes);
6466

H
Haojun Liao 已提交
6467
    if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) {
6468 6469 6470
      doSetOperatorCompleted(pOperator);
    }

H
Haojun Liao 已提交
6471
    return pIntervalInfo->binfo.pRes;
6472 6473
  }

H
Haojun Liao 已提交
6474
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
6475 6476 6477
  int32_t order = pQueryAttr->order.order;
  STimeWindow win = pQueryAttr->window;

H
Haojun Liao 已提交
6478
  SOperatorInfo* downstream = pOperator->pDownstream[0];
6479 6480

  while(1) {
H
Haojun Liao 已提交
6481 6482 6483
    publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
    publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
6484 6485 6486 6487 6488

    if (pBlock == NULL) {
      break;
    }

H
Haojun Liao 已提交
6489
//    setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput);
6490 6491

    // the pDataBlock are always the same one, no need to call this again
H
Haojun Liao 已提交
6492 6493
    setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order);
    hashAllIntervalAgg(pOperator, &pIntervalInfo->binfo.resultRowInfo, pBlock, 0);
6494 6495 6496 6497 6498 6499 6500
  }

  // restore the value
  pQueryAttr->order.order = order;
  pQueryAttr->window = win;

  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
6501
  closeAllResultRows(&pIntervalInfo->binfo.resultRowInfo);
6502
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
6503
  finalizeQueryResult(pOperator, pIntervalInfo->binfo.pCtx, &pIntervalInfo->binfo.resultRowInfo, pIntervalInfo->binfo.rowCellInfoOffset);
6504

H
Haojun Liao 已提交
6505
  initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->binfo.resultRowInfo);
H
Haojun Liao 已提交
6506
//  toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes);
6507

H
Haojun Liao 已提交
6508
  if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) {
6509 6510 6511
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
6512
  return pIntervalInfo->binfo.pRes->info.rows == 0? NULL:pIntervalInfo->binfo.pRes;
6513 6514 6515 6516 6517 6518 6519 6520 6521
}

static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  STableIntervalOperatorInfo* pIntervalInfo = pOperator->info;
H
Haojun Liao 已提交
6522
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
6523 6524 6525 6526

  if (pOperator->status == OP_RES_TO_RETURN) {
    int64_t st = taosGetTimestampUs();

H
Haojun Liao 已提交
6527
//    copyToSDataBlock(NULL, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset);
H
Haojun Liao 已提交
6528
    if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) {
6529 6530 6531 6532 6533 6534
      doSetOperatorCompleted(pOperator);
    }

    SQInfo* pQInfo = pRuntimeEnv->qinfo;
    pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st);

H
Haojun Liao 已提交
6535
    return pIntervalInfo->binfo.pRes;
6536 6537
  }

H
Haojun Liao 已提交
6538
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
6539 6540
  int32_t order = pQueryAttr->order.order;

H
Haojun Liao 已提交
6541
  SOperatorInfo* downstream = pOperator->pDownstream[0];
6542 6543

  while(1) {
H
Haojun Liao 已提交
6544 6545 6546
    publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
    publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
6547 6548 6549 6550 6551 6552 6553 6554

    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
    STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;

H
Haojun Liao 已提交
6555
//    setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput);
H
Haojun Liao 已提交
6556
    setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order);
6557 6558 6559 6560 6561 6562 6563 6564
    setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey);

    hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex);
  }

  pOperator->status = OP_RES_TO_RETURN;
  pQueryAttr->order.order = order;   // TODO : restore the order
  doCloseAllTimeWindow(pRuntimeEnv);
6565
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
6566

H
Haojun Liao 已提交
6567
//  copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset);
H
Haojun Liao 已提交
6568
  if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) {
6569 6570 6571
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
6572
  return pIntervalInfo->binfo.pRes;
6573 6574 6575 6576 6577 6578 6579 6580 6581
}

static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  STableIntervalOperatorInfo* pIntervalInfo = pOperator->info;
H
Haojun Liao 已提交
6582
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
6583 6584

  if (pOperator->status == OP_RES_TO_RETURN) {
H
Haojun Liao 已提交
6585
//    copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset);
H
Haojun Liao 已提交
6586
    if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) {
6587 6588 6589
      pOperator->status = OP_EXEC_DONE;
    }

H
Haojun Liao 已提交
6590
    return pIntervalInfo->binfo.pRes;
6591 6592
  }

H
Haojun Liao 已提交
6593
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
6594 6595
  int32_t order = pQueryAttr->order.order;

H
Haojun Liao 已提交
6596
  SOperatorInfo* downstream = pOperator->pDownstream[0];
6597 6598

  while(1) {
H
Haojun Liao 已提交
6599 6600 6601
    publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
    publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
6602 6603 6604 6605 6606 6607 6608 6609

    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
    STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;

H
Haojun Liao 已提交
6610
//    setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput);
H
Haojun Liao 已提交
6611
    setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order);
6612 6613 6614 6615 6616 6617 6618 6619
    setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey);

    hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex);
  }

  pOperator->status = OP_RES_TO_RETURN;
  pQueryAttr->order.order = order;   // TODO : restore the order
  doCloseAllTimeWindow(pRuntimeEnv);
6620
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
6621 6622

  int64_t st = taosGetTimestampUs();
H
Haojun Liao 已提交
6623
//  copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset);
H
Haojun Liao 已提交
6624
  if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) {
6625 6626 6627 6628 6629 6630
    pOperator->status = OP_EXEC_DONE;
  }

  SQInfo* pQInfo = pRuntimeEnv->qinfo;
  pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st);

H
Haojun Liao 已提交
6631
  return pIntervalInfo->binfo.pRes;
6632 6633 6634
}

static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) {
H
Haojun Liao 已提交
6635
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
6636 6637 6638 6639 6640
  STableQueryInfo*  item = pRuntimeEnv->current;
  SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex);

  SOptrBasicInfo* pBInfo = &pInfo->binfo;

H
Haojun Liao 已提交
6641
  bool    masterScan = IS_MAIN_SCAN(pRuntimeEnv);
6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682
  int16_t     bytes = pColInfoData->info.bytes;
  int16_t     type = pColInfoData->info.type;

  SColumnInfoData* pTsColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0);
  TSKEY* tsList = (TSKEY*)pTsColInfoData->pData;
  if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) {
    pInfo->reptScan = true;
    tfree(pInfo->prevData);
  }

  pInfo->numOfRows = 0;
  for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) {
    char* val = ((char*)pColInfoData->pData) + bytes * j;
    if (isNull(val, type)) {
      continue;
    }
    if (pInfo->prevData == NULL) {
      pInfo->prevData = malloc(bytes);
      memcpy(pInfo->prevData, val, bytes);
      pInfo->numOfRows = 1;
      pInfo->curWindow.skey = tsList[j];
      pInfo->curWindow.ekey = tsList[j];
      pInfo->start = j;

    } else if (memcmp(pInfo->prevData, val, bytes) == 0) {
      pInfo->curWindow.ekey = tsList[j];
      pInfo->numOfRows += 1;
      //pInfo->start = j;
      if (j == 0 && pInfo->start != 0) {
        pInfo->numOfRows = 1;
        pInfo->start = 0;
      }
    } else {
      SResultRow* pResult = NULL;
      pInfo->curWindow.ekey = pInfo->curWindow.skey;
      int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan,
                                            &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput,
                                            pBInfo->rowCellInfoOffset);
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
      }
H
Haojun Liao 已提交
6683 6684
//      doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList,
//                       pSDataBlock->info.rows, pOperator->numOfOutput);
6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704

      pInfo->curWindow.skey = tsList[j];
      pInfo->curWindow.ekey = tsList[j];
      memcpy(pInfo->prevData, val, bytes);
      pInfo->numOfRows = 1;
      pInfo->start = j;

    }
  }

  SResultRow* pResult = NULL;

  pInfo->curWindow.ekey = pInfo->curWindow.skey;
  int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan,
                                        &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput,
                                        pBInfo->rowCellInfoOffset);
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
  }

H
Haojun Liao 已提交
6705 6706
//  doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList,
//                   pSDataBlock->info.rows, pOperator->numOfOutput);
6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717
}

static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pWindowInfo = pOperator->info;
  SOptrBasicInfo* pBInfo = &pWindowInfo->binfo;

H
Haojun Liao 已提交
6718
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
6719
  if (pOperator->status == OP_RES_TO_RETURN) {
H
Haojun Liao 已提交
6720
//    toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes);
6721 6722 6723 6724 6725 6726 6727 6728

    if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) {
      pOperator->status = OP_EXEC_DONE;
    }

    return pBInfo->pRes;
  }

H
Haojun Liao 已提交
6729
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
6730 6731
  int32_t order = pQueryAttr->order.order;
  STimeWindow win = pQueryAttr->window;
H
Haojun Liao 已提交
6732
  SOperatorInfo* downstream = pOperator->pDownstream[0];
6733
  while (1) {
H
Haojun Liao 已提交
6734 6735 6736
    publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
    publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753

    if (pBlock == NULL) {
      break;
    }
    setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order);
    if (pWindowInfo->colIndex == -1) {
      pWindowInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock);
    }
    doStateWindowAggImpl(pOperator,  pWindowInfo, pBlock);
  }

  // restore the value
  pQueryAttr->order.order = order;
  pQueryAttr->window = win;

  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pBInfo->resultRowInfo);
6754
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
6755 6756 6757
  finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset);

  initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo);
H
Haojun Liao 已提交
6758
//  toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes);
6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776

  if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) {
    pOperator->status = OP_EXEC_DONE;
  }

  return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes;
}

static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSWindowOperatorInfo* pWindowInfo = pOperator->info;
  SOptrBasicInfo* pBInfo = &pWindowInfo->binfo;


H
Haojun Liao 已提交
6777
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
6778
  if (pOperator->status == OP_RES_TO_RETURN) {
H
Haojun Liao 已提交
6779
//    toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes);
6780 6781 6782 6783 6784 6785 6786 6787

    if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) {
      pOperator->status = OP_EXEC_DONE;
    }

    return pBInfo->pRes;
  }

H
Haojun Liao 已提交
6788
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
6789 6790 6791 6792
  //pQueryAttr->order.order = TSDB_ORDER_ASC;
  int32_t order = pQueryAttr->order.order;
  STimeWindow win = pQueryAttr->window;

H
Haojun Liao 已提交
6793
  SOperatorInfo* downstream = pOperator->pDownstream[0];
6794 6795

  while(1) {
H
Haojun Liao 已提交
6796 6797 6798
    publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
    publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order);
    doSessionWindowAggImpl(pOperator, pWindowInfo, pBlock);
  }

  // restore the value
  pQueryAttr->order.order = order;
  pQueryAttr->window = win;

  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pBInfo->resultRowInfo);
H
Haojun Liao 已提交
6814
//  setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED);
6815 6816 6817
  finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset);

  initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo);
H
Haojun Liao 已提交
6818
//  toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes);
6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834

  if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) {
    pOperator->status = OP_EXEC_DONE;
  }

  return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes;
}

static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SGroupbyOperatorInfo *pInfo = pOperator->info;

H
Haojun Liao 已提交
6835
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
6836
  if (pOperator->status == OP_RES_TO_RETURN) {
H
Haojun Liao 已提交
6837
//    toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes);
6838 6839 6840 6841 6842 6843 6844 6845

    if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) {
      pOperator->status = OP_EXEC_DONE;
    }

    return pInfo->binfo.pRes;
  }

H
Haojun Liao 已提交
6846
  SOperatorInfo* downstream = pOperator->pDownstream[0];
6847 6848

  while(1) {
H
Haojun Liao 已提交
6849 6850 6851
    publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
    publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
6852 6853 6854 6855 6856 6857
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pRuntimeEnv->pQueryAttr->order.order);
H
Haojun Liao 已提交
6858
//    setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput);
6859 6860 6861 6862 6863 6864 6865 6866 6867
    if (pInfo->colIndex == -1) {
      pInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock);
    }

    doHashGroupbyAgg(pOperator, pInfo, pBlock);
  }

  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pInfo->binfo.resultRowInfo);
H
Haojun Liao 已提交
6868
//  setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED);
6869 6870 6871 6872

  if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows
    finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset);
  } else {
6873
    updateNumOfRowsInResultRows(pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset);
6874 6875 6876 6877 6878 6879 6880
  }

  initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->binfo.resultRowInfo);
  if (!pRuntimeEnv->pQueryAttr->stableQuery) {
    sortGroupResByOrderList(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes);
  }

H
Haojun Liao 已提交
6881
//  toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes);
6882 6883 6884 6885 6886 6887 6888
  if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) {
    pOperator->status = OP_EXEC_DONE;
  }

  return pInfo->binfo.pRes;
}

H
Haojun Liao 已提交
6889
static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, STaskRuntimeEnv* pRuntimeEnv, bool* newgroup) {
6890
  pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows;
6891
  int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey;
6892 6893 6894 6895 6896 6897 6898 6899 6900 6901
  taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo));

  taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey);
  taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->existNewGroupBlock);

  doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p);
  pInfo->existNewGroupBlock = NULL;
  *newgroup = true;
}

H
Haojun Liao 已提交
6902
static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, STaskRuntimeEnv  *pRuntimeEnv, bool *newgroup) {
6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926
  if (taosFillHasMoreResults(pInfo->pFillInfo)) {
    *newgroup = false;
    doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pRuntimeEnv->resultInfo.capacity, pInfo->p);
    if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult)) {
      return;
    }
  }

  // handle the cached new group data block
  if (pInfo->existNewGroupBlock) {
    doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup);
  }
}

static SSDataBlock* doFill(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;

  SFillOperatorInfo *pInfo = pOperator->info;
  pInfo->pRes->info.rows = 0;

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

H
Haojun Liao 已提交
6927
  STaskRuntimeEnv  *pRuntimeEnv = pOperator->pRuntimeEnv;
6928 6929 6930 6931 6932 6933
  doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup);
  if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) {
    return pInfo->pRes;
  }

  while(1) {
H
Haojun Liao 已提交
6934 6935 6936
    publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
    SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
    publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991

    if (*newgroup) {
      assert(pBlock != NULL);
    }

    if (*newgroup && pInfo->totalInputRows > 0) {  // there are already processed current group data block
      pInfo->existNewGroupBlock = pBlock;
      *newgroup = false;

      // Fill the previous group data block, before handle the data block of new group.
      // Close the fill operation for previous group data block
      taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey);
    } else {
      if (pBlock == NULL) {
        if (pInfo->totalInputRows == 0) {
          pOperator->status = OP_EXEC_DONE;
          return NULL;
        }

        taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey);
      } else {
        pInfo->totalInputRows += pBlock->info.rows;
        taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, pBlock->info.window.ekey);
        taosFillSetInputDataBlock(pInfo->pFillInfo, pBlock);
      }
    }

    doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p);

    // current group has no more result to return
    if (pInfo->pRes->info.rows > 0) {
      // 1. The result in current group not reach the threshold of output result, continue
      // 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately
      if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL || (!pInfo->multigroupResult)) {
        return pInfo->pRes;
      }

      doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup);
      if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL) {
        return pInfo->pRes;
      }
    } else if (pInfo->existNewGroupBlock) {  // try next group
      assert(pBlock != NULL);
      doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup);

      if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold) {
        return pInfo->pRes;
      }
    } else {
      return NULL;
    }
  }
}

// todo set the attribute of query scan count
H
Haojun Liao 已提交
6992
static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr) {
6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003 7004 7005 7006 7007
  for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
    int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]);
    if (functionId == FUNCTION_STDDEV || functionId == FUNCTION_PERCT) {
      return 2;
    }
  }

  return 1;
}

static void destroyOperatorInfo(SOperatorInfo* pOperator) {
  if (pOperator == NULL) {
    return;
  }

H
Haojun Liao 已提交
7008 7009
  if (pOperator->cleanupFn != NULL) {
    pOperator->cleanupFn(pOperator->info, pOperator->numOfOutput);
7010 7011
  }

H
Haojun Liao 已提交
7012
  if (pOperator->pDownstream != NULL) {
H
Haojun Liao 已提交
7013
    for(int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
H
Haojun Liao 已提交
7014
      destroyOperatorInfo(pOperator->pDownstream[i]);
7015 7016
    }

H
Haojun Liao 已提交
7017
    tfree(pOperator->pDownstream);
H
Haojun Liao 已提交
7018
    pOperator->numOfDownstream = 0;
7019 7020 7021 7022 7023 7024
  }

  tfree(pOperator->info);
  tfree(pOperator);
}

7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049
static int32_t initAggSup(SAggSupporter* pAggSup, SArray* pExprInfo) {
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

  pAggSup->keyBuf              = calloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES);
  pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK);
  pAggSup->pResultRowListSet   = taosHashInit(100, hashFn, false, HASH_NO_LOCK);
  pAggSup->pool                = initResultRowPool(getResultRowSize(pExprInfo));
  pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell));

  if (pAggSup->keyBuf == NULL || pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL ||
      pAggSup->pResultRowHashTable == NULL || pAggSup->pool == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  return TSDB_CODE_SUCCESS;
}

static void clearupAggSup(SAggSupporter* pAggSup) {
  tfree(pAggSup->keyBuf);
  taosHashCleanup(pAggSup->pResultRowHashTable);
  taosHashCleanup(pAggSup->pResultRowListSet);
  taosArrayDestroy(pAggSup->pResultRowArrayList);
  destroyResultRowPool(pAggSup->pool);
}

H
Haojun Liao 已提交
7050
static int32_t initAggInfo(SAggOperatorInfo* pInfo, SArray* pExprInfo, int32_t numOfRows, const STableGroupInfo* pTableGroupInfo) {
H
Haojun Liao 已提交
7051
  pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, numOfRows);
H
Haojun Liao 已提交
7052
  pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize);
H
Haojun Liao 已提交
7053
  pInfo->binfo.capacity = 4096;
H
Haojun Liao 已提交
7054

7055
  initAggSup(&pInfo->aggSup, pExprInfo);
H
Haojun Liao 已提交
7056 7057 7058 7059 7060 7061 7062 7063
  pInfo->pTableQueryInfo = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo));

  int32_t index = 0;
  for(int32_t i = 0; i < taosArrayGetSize(pTableGroupInfo->pGroupList); ++i) {
    SArray* pa = taosArrayGetP(pTableGroupInfo->pGroupList, i);
    for(int32_t j = 0; j < taosArrayGetSize(pa); ++j) {
      STableKeyInfo* pk = taosArrayGet(pa, j);

H
Haojun Liao 已提交
7064
      STableQueryInfo* pTQueryInfo = &pInfo->pTableQueryInfo[index++];
H
Haojun Liao 已提交
7065 7066 7067 7068 7069
      pTQueryInfo->uid        = pk->uid;
      pTQueryInfo->lastKey    = pk->lastKey;
      pTQueryInfo->groupIndex = i;
    }
  }
H
Haojun Liao 已提交
7070 7071 7072 7073 7074

  STimeWindow win = {0, INT64_MAX};
  createTableQueryInfo(pInfo->pTableQueryInfo, false, win);

  return TSDB_CODE_SUCCESS;
H
Haojun Liao 已提交
7075 7076
}

H
Haojun Liao 已提交
7077
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) {
H
Haojun Liao 已提交
7078 7079 7080 7081 7082
  SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo));

  int32_t numOfRows = 1;
  //(int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery));

H
Haojun Liao 已提交
7083
  initAggInfo(pInfo, pExprInfo, numOfRows, pTableGroupInfo);
7084
  setDefaultOutputBuf_rv(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, pTaskInfo);
7085 7086 7087

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  pOperator->name         = "TableAggregate";
X
Xiaoyu Wang 已提交
7088
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_AGG;
7089 7090 7091
  pOperator->blockingOptr = true;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->info         = pInfo;
H
Haojun Liao 已提交
7092 7093
  pOperator->pExpr        = exprArrayDup(pExprInfo);
  pOperator->numOfOutput  = taosArrayGetSize(pExprInfo);
7094

H
Haojun Liao 已提交
7095
  pOperator->pTaskInfo    = pTaskInfo;
7096
  pOperator->exec         = doAggregate;
H
Haojun Liao 已提交
7097
  pOperator->cleanupFn    = destroyAggOperatorInfo;
7098
  int32_t code = appendDownstream(pOperator, &downstream, 1);
7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109

  return pOperator;
}

static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) {
  assert(pInfo != NULL);

  destroySQLFunctionCtx(pInfo->pCtx, numOfOutput);
  tfree(pInfo->rowCellInfoOffset);

  cleanupResultRowInfo(&pInfo->resultRowInfo);
H
Haojun Liao 已提交
7110
  pInfo->pRes = blockDataDestroy(pInfo->pRes);
7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133
}

static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) {
  SOptrBasicInfo* pInfo = (SOptrBasicInfo*) param;
  doDestroyBasicInfo(pInfo, numOfOutput);
}
static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*) param;
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  tfree(pInfo->prevData);
}
static void destroyAggOperatorInfo(void* param, int32_t numOfOutput) {
  SAggOperatorInfo* pInfo = (SAggOperatorInfo*) param;
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
}
static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SSWindowOperatorInfo* pInfo = (SSWindowOperatorInfo*) param;
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
}

static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) {
  SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param;
  pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo);
H
Haojun Liao 已提交
7134
  pInfo->pRes = blockDataDestroy(pInfo->pRes);
7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150
  tfree(pInfo->p);
}

static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) {
  SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*) param;
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  tfree(pInfo->prevData);
}

static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
  SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*) param;
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
}

static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
  STagScanInfo* pInfo = (STagScanInfo*) param;
H
Haojun Liao 已提交
7151
  pInfo->pRes = blockDataDestroy(pInfo->pRes);
7152 7153 7154 7155
}

static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
  SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param;
H
Haojun Liao 已提交
7156 7157
  pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock);

7158
  taosArrayDestroy(pInfo->orderInfo);
7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170
}

static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) {
  SFilterOperatorInfo* pInfo = (SFilterOperatorInfo*) param;
  doDestroyFilterInfo(pInfo->pFilterInfo, pInfo->numOfFilterCols);
}

static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) {
  SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*) param;
  taosHashCleanup(pInfo->pSet);
  tfree(pInfo->buf);
  taosArrayDestroy(pInfo->pDistinctDataInfo);
H
Haojun Liao 已提交
7171
  pInfo->pRes = blockDataDestroy(pInfo->pRes);
7172 7173
}

H
Haojun Liao 已提交
7174
SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) {
7175 7176
  SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo));

H
Haojun Liao 已提交
7177 7178
  int32_t numOfRows = 1;
  size_t  numOfOutput = taosArrayGetSize(pExprInfo);
H
Haojun Liao 已提交
7179 7180 7181
  initAggInfo(pInfo, pExprInfo, numOfRows, pTableGroupInfo);

  size_t  tableGroup = taosArrayGetSize(pTableGroupInfo->pGroupList);
7182
  initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup);
7183 7184 7185

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  pOperator->name         = "MultiTableAggregate";
X
Xiaoyu Wang 已提交
7186
  // pOperator->operatorType = OP_MultiTableAggregate;
7187 7188 7189
  pOperator->blockingOptr = true;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->info         = pInfo;
H
Haojun Liao 已提交
7190
  pOperator->pExpr        = exprArrayDup(pExprInfo);
7191 7192
  pOperator->numOfOutput  = numOfOutput;

H
Haojun Liao 已提交
7193
  pOperator->exec         = doMultiTableAggregate;
H
Haojun Liao 已提交
7194
  pOperator->cleanupFn    = destroyAggOperatorInfo;
7195
  int32_t code = appendDownstream(pOperator, &downstream, 1);
7196 7197 7198 7199

  return pOperator;
}

H
Haojun Liao 已提交
7200
SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
7201 7202 7203 7204 7205 7206 7207
  SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo));

  pInfo->seed = rand();
  pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity;

  SOptrBasicInfo* pBInfo = &pInfo->binfo;
  pBInfo->pRes  = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity);
H
Haojun Liao 已提交
7208
  pBInfo->pCtx  = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset);
7209

7210
  initResultRowInfo(&pBInfo->resultRowInfo, 8);
H
Haojun Liao 已提交
7211
  setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN);
7212 7213 7214

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  pOperator->name         = "ProjectOperator";
H
Haojun Liao 已提交
7215
//  pOperator->operatorType = OP_Project;
7216 7217 7218 7219 7220 7221 7222 7223
  pOperator->blockingOptr = false;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->info         = pInfo;
  pOperator->pExpr        = pExpr;
  pOperator->numOfOutput  = numOfOutput;
  pOperator->pRuntimeEnv  = pRuntimeEnv;

  pOperator->exec         = doProjectOperation;
H
Haojun Liao 已提交
7224
  pOperator->cleanupFn    = destroyProjectOperatorInfo;
7225
  int32_t code = appendDownstream(pOperator, &downstream, 1);
7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262

  return pOperator;
}

SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols) {
#if 0
  SColumnInfo* pCols = calloc(numOfOutput, sizeof(SColumnInfo));

  int32_t numOfFilter = 0;
  for(int32_t i = 0; i < numOfOutput; ++i) {
    if (pExpr[i].base.flist.numOfFilters > 0) {
      numOfFilter += 1;
    }

    pCols[i].type  = pExpr[i].base.resSchema.type;
    pCols[i].bytes = pExpr[i].base.resSchema.bytes;
    pCols[i].colId = pExpr[i].base.resSchema.colId;

    pCols[i].flist.numOfFilters = pExpr[i].base.flist.numOfFilters;
    if (pCols[i].flist.numOfFilters != 0) { 
      pCols[i].flist.filterInfo   = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo));
      memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo));
    } else {
      // avoid runtime error
      pCols[i].flist.filterInfo   = NULL; 
    }
  }

  assert(numOfFilter > 0);

  *numOfFilterCols = numOfFilter;
  return pCols;
#endif

  return 0;
}

H
Haojun Liao 已提交
7263
SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
7264 7265 7266 7267
                                        int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter) {
  SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo));

  assert(numOfFilter > 0 && pCols != NULL);
7268
//  doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0);
7269 7270 7271 7272 7273
  pInfo->numOfFilterCols = numOfFilter;

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));

  pOperator->name         = "FilterOperator";
H
Haojun Liao 已提交
7274
//  pOperator->operatorType = OP_Filter;
7275 7276 7277 7278 7279 7280 7281
  pOperator->blockingOptr = false;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->numOfOutput  = numOfOutput;
  pOperator->pExpr        = pExpr;
  pOperator->exec         = doFilter;
  pOperator->info         = pInfo;
  pOperator->pRuntimeEnv  = pRuntimeEnv;
H
Haojun Liao 已提交
7282
  pOperator->cleanupFn    = destroyConditionOperatorInfo;
7283
  int32_t code = appendDownstream(pOperator, &downstream, 1);
7284 7285 7286 7287

  return pOperator;
}

H
Haojun Liao 已提交
7288
SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream) {
7289 7290 7291 7292 7293 7294
  SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo));
  pInfo->limit = pRuntimeEnv->pQueryAttr->limit.limit;

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));

  pOperator->name         = "LimitOperator";
H
Haojun Liao 已提交
7295
//  pOperator->operatorType = OP_Limit;
7296 7297 7298 7299 7300
  pOperator->blockingOptr = false;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->exec         = doLimit;
  pOperator->info         = pInfo;
  pOperator->pRuntimeEnv  = pRuntimeEnv;
7301
    int32_t code = appendDownstream(pOperator, &downstream, 1);
7302 7303 7304 7305

  return pOperator;
}

H
Haojun Liao 已提交
7306
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) {
7307 7308
  STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));

H
Haojun Liao 已提交
7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327
  initAggSup(&pInfo->aggSup, pExprInfo);

  // todo:
  pInfo->order = TSDB_ORDER_ASC;
  pInfo->precision = TSDB_TIME_PRECISION_MICRO;
  pInfo->win.skey = INT64_MIN;
  pInfo->win.ekey = INT64_MAX;
  pInfo->interval.intervalUnit = 's';
  pInfo->interval.slidingUnit  = 's';
  pInfo->interval.interval = 1000;
  pInfo->interval.sliding  = 1000;

  int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, 0, "/tmp/");

  int32_t numOfOutput = taosArrayGetSize(pExprInfo);
  pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize);
  pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity);

  initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1);
7328 7329 7330 7331

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));

  pOperator->name         = "TimeIntervalAggOperator";
X
Xiaoyu Wang 已提交
7332
  // pOperator->operatorType = OP_TimeWindow;
7333 7334
  pOperator->blockingOptr = true;
  pOperator->status       = OP_IN_EXECUTING;
H
Haojun Liao 已提交
7335 7336 7337
  pOperator->pExpr        = exprArrayDup(pExprInfo);

  pOperator->pTaskInfo    = pTaskInfo;
7338 7339 7340
  pOperator->numOfOutput  = numOfOutput;
  pOperator->info         = pInfo;
  pOperator->exec         = doIntervalAgg;
H
Haojun Liao 已提交
7341
  pOperator->cleanupFn    = destroyBasicOperatorInfo;
7342

H
Haojun Liao 已提交
7343
  code = appendDownstream(pOperator, &downstream, 1);
7344 7345 7346 7347
  return pOperator;
}


H
Haojun Liao 已提交
7348
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
7349 7350
  STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));

H
Haojun Liao 已提交
7351 7352 7353
  pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
  pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
  initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
7354 7355 7356 7357

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));

  pOperator->name         = "AllTimeIntervalAggOperator";
H
Haojun Liao 已提交
7358
//  pOperator->operatorType = OP_AllTimeWindow;
7359 7360 7361 7362 7363 7364 7365
  pOperator->blockingOptr = true;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->pExpr        = pExpr;
  pOperator->numOfOutput  = numOfOutput;
  pOperator->info         = pInfo;
  pOperator->pRuntimeEnv  = pRuntimeEnv;
  pOperator->exec         = doAllIntervalAgg;
H
Haojun Liao 已提交
7366
  pOperator->cleanupFn    = destroyBasicOperatorInfo;
7367

7368
    int32_t code = appendDownstream(pOperator, &downstream, 1);
7369 7370 7371
  return pOperator;
}

H
Haojun Liao 已提交
7372
SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
7373 7374 7375
  SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo));
  pInfo->colIndex   = -1;
  pInfo->reptScan   = false;
H
Haojun Liao 已提交
7376
  pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
7377
  pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
7378
  initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
7379 7380 7381

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  pOperator->name         = "StateWindowOperator";
H
Haojun Liao 已提交
7382
//  pOperator->operatorType = OP_StateWindow;
7383 7384 7385 7386 7387 7388 7389
  pOperator->blockingOptr = true;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->pExpr        = pExpr;
  pOperator->numOfOutput  = numOfOutput;
  pOperator->info         = pInfo;
  pOperator->pRuntimeEnv  = pRuntimeEnv;
  pOperator->exec         = doStateWindowAgg;
H
Haojun Liao 已提交
7390
  pOperator->cleanupFn    = destroyStateWindowOperatorInfo;
7391

7392
    int32_t code = appendDownstream(pOperator, &downstream, 1);
7393 7394
  return pOperator;
}
H
Haojun Liao 已提交
7395
SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
7396 7397
  SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo));

H
Haojun Liao 已提交
7398
  pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
7399
  pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
7400
  initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
7401 7402 7403 7404 7405 7406

  pInfo->prevTs   = INT64_MIN;
  pInfo->reptScan = false;
  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));

  pOperator->name         = "SessionWindowAggOperator";
H
Haojun Liao 已提交
7407
//  pOperator->operatorType = OP_SessionWindow;
7408 7409 7410 7411 7412 7413 7414
  pOperator->blockingOptr = true;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->pExpr        = pExpr;
  pOperator->numOfOutput  = numOfOutput;
  pOperator->info         = pInfo;
  pOperator->pRuntimeEnv  = pRuntimeEnv;
  pOperator->exec         = doSessionWindowAgg;
H
Haojun Liao 已提交
7415
  pOperator->cleanupFn    = destroySWindowOperatorInfo;
7416

7417
    int32_t code = appendDownstream(pOperator, &downstream, 1);
7418 7419 7420
  return pOperator;
}

H
Haojun Liao 已提交
7421
SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
7422 7423
  STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));

H
Haojun Liao 已提交
7424 7425 7426
  pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
  pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
  initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
7427 7428 7429

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  pOperator->name         = "MultiTableTimeIntervalOperator";
H
Haojun Liao 已提交
7430
//  pOperator->operatorType = OP_MultiTableTimeInterval;
7431 7432 7433 7434 7435 7436 7437 7438
  pOperator->blockingOptr = true;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->pExpr        = pExpr;
  pOperator->numOfOutput  = numOfOutput;
  pOperator->info         = pInfo;
  pOperator->pRuntimeEnv  = pRuntimeEnv;

  pOperator->exec         = doSTableIntervalAgg;
H
Haojun Liao 已提交
7439
  pOperator->cleanupFn    = destroyBasicOperatorInfo;
7440

7441
    int32_t code = appendDownstream(pOperator, &downstream, 1);
7442 7443 7444
  return pOperator;
}

H
Haojun Liao 已提交
7445
SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
7446 7447
  STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));

H
Haojun Liao 已提交
7448 7449 7450
  pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
  pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
  initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
7451 7452 7453

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  pOperator->name         = "AllMultiTableTimeIntervalOperator";
H
Haojun Liao 已提交
7454
//  pOperator->operatorType = OP_AllMultiTableTimeInterval;
7455 7456 7457 7458 7459 7460 7461 7462
  pOperator->blockingOptr = true;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->pExpr        = pExpr;
  pOperator->numOfOutput  = numOfOutput;
  pOperator->info         = pInfo;
  pOperator->pRuntimeEnv  = pRuntimeEnv;

  pOperator->exec         = doAllSTableIntervalAgg;
H
Haojun Liao 已提交
7463
  pOperator->cleanupFn    = destroyBasicOperatorInfo;
7464

7465
    int32_t code = appendDownstream(pOperator, &downstream, 1);
7466 7467 7468 7469

  return pOperator;
}

H
Haojun Liao 已提交
7470
SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
7471 7472 7473 7474
  SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo));
  pInfo->colIndex = -1;  // group by column index


H
Haojun Liao 已提交
7475
  pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
7476

H
Haojun Liao 已提交
7477
  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
7478 7479 7480 7481 7482

  pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize *
      (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)));

  pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
7483
  initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
7484 7485 7486 7487 7488

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  pOperator->name         = "GroupbyAggOperator";
  pOperator->blockingOptr = true;
  pOperator->status       = OP_IN_EXECUTING;
H
Haojun Liao 已提交
7489
//  pOperator->operatorType = OP_Groupby;
7490 7491 7492 7493 7494
  pOperator->pExpr        = pExpr;
  pOperator->numOfOutput  = numOfOutput;
  pOperator->info         = pInfo;
  pOperator->pRuntimeEnv  = pRuntimeEnv;
  pOperator->exec         = hashGroupbyAggregate;
H
Haojun Liao 已提交
7495
  pOperator->cleanupFn    = destroyGroupbyOperatorInfo;
7496

7497
    int32_t code = appendDownstream(pOperator, &downstream, 1);
7498 7499 7500
  return pOperator;
}

H
Haojun Liao 已提交
7501
SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) {
7502 7503 7504 7505 7506
  SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo));
  pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
  pInfo->multigroupResult = multigroupResult;

  {
H
Haojun Liao 已提交
7507
    STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
7508 7509 7510
    struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal);
    STimeWindow w = TSWINDOW_INITIALIZER;

dengyihao's avatar
dengyihao 已提交
7511 7512
    TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
    TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
H
Haojun Liao 已提交
7513
//    getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w);
7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527

    pInfo->pFillInfo =
        taosCreateFillInfo(pQueryAttr->order.order, w.skey, 0, (int32_t)pRuntimeEnv->resultInfo.capacity, numOfOutput,
                           pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit,
                           (int8_t)pQueryAttr->precision, pQueryAttr->fillType, pColInfo, pRuntimeEnv->qinfo);

    pInfo->p = calloc(numOfOutput, POINTER_BYTES);
  }

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));

  pOperator->name         = "FillOperator";
  pOperator->blockingOptr = false;
  pOperator->status       = OP_IN_EXECUTING;
H
Haojun Liao 已提交
7528
//  pOperator->operatorType = OP_Fill;
7529 7530 7531 7532 7533
  pOperator->pExpr        = pExpr;
  pOperator->numOfOutput  = numOfOutput;
  pOperator->info         = pInfo;
  pOperator->pRuntimeEnv  = pRuntimeEnv;
  pOperator->exec         = doFill;
H
Haojun Liao 已提交
7534
  pOperator->cleanupFn    = destroySFillOperatorInfo;
7535

7536
    int32_t code = appendDownstream(pOperator, &downstream, 1);
7537 7538 7539
  return pOperator;
}

H
Haojun Liao 已提交
7540
SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) {
7541 7542
  SSLimitOperatorInfo* pInfo = calloc(1, sizeof(SSLimitOperatorInfo));

H
Haojun Liao 已提交
7543
  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575

  pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr);
  pInfo->slimit          = pQueryAttr->slimit;
  pInfo->limit           = pQueryAttr->limit;
  pInfo->capacity        = pRuntimeEnv->resultInfo.capacity;
  pInfo->threshold       = (int64_t)(pInfo->capacity * 0.8);
  pInfo->currentOffset   = pQueryAttr->limit.offset;
  pInfo->currentGroupOffset = pQueryAttr->slimit.offset;
  pInfo->multigroupResult= multigroupResult;

  // TODO refactor
  int32_t len = 0;
  for(int32_t i = 0; i < numOfOutput; ++i) {
    len += pExpr[i].base.resSchema.bytes;
  }

  int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0;
  pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len));

  int32_t offset = POINTER_BYTES * numOfCols;
  for(int32_t i = 0; i < numOfCols; ++i) {
    pInfo->prevRow[i] = (char*)pInfo->prevRow + offset;

    SColIndex* index = taosArrayGet(pInfo->orderColumnList, i);
    offset += pExpr[index->colIndex].base.resSchema.bytes;
  }

  pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));

  pOperator->name         = "SLimitOperator";
X
Xiaoyu Wang 已提交
7576
  // pOperator->operatorType = OP_SLimit;
7577 7578
  pOperator->blockingOptr = false;
  pOperator->status       = OP_IN_EXECUTING;
7579
//  pOperator->exec         = doSLimit;
7580 7581
  pOperator->info         = pInfo;
  pOperator->pRuntimeEnv  = pRuntimeEnv;
H
Haojun Liao 已提交
7582
  pOperator->cleanupFn    = destroySlimitOperatorInfo;
7583

7584
    int32_t code = appendDownstream(pOperator, &downstream, 1);
7585 7586 7587 7588 7589 7590 7591 7592 7593 7594
  return pOperator;
}

static SSDataBlock* doTagScan(void* param, bool* newgroup) {
#if 0
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

H
Haojun Liao 已提交
7595
  STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv;
7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606
  int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity;

  STagScanInfo *pInfo = pOperator->info;
  SSDataBlock  *pRes = pInfo->pRes;
  *newgroup = false;

  int32_t count = 0;
  SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0);

  int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]);
  if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id
H
Haojun Liao 已提交
7607
    STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618
    assert(pQueryAttr->numOfOutput == 1);

    SExprInfo* pExprInfo = &pOperator->pExpr[0];
    int32_t rsize = pExprInfo->base.resSchema.bytes;

    count = 0;

    int16_t bytes = pExprInfo->base.resSchema.bytes;
    int16_t type  = pExprInfo->base.resSchema.type;

    for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) {
7619
      if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) {
7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650
        bytes = pQueryAttr->tagColList[i].bytes;
        type = pQueryAttr->tagColList[i].type;
        break;
      }
    }

    SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0);

    while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
      int32_t i = pInfo->curPos++;
      STableQueryInfo *item = taosArrayGetP(pa, i);

      char *output = pColInfo->pData + count * rsize;
      varDataSetLen(output, rsize - VARSTR_HEADER_SIZE);

      output = varDataVal(output);
      STableId* id = TSDB_TABLEID(item->pTable);

      *(int16_t *)output = 0;
      output += sizeof(int16_t);

      *(int64_t *)output = id->uid;  // memory align problem, todo serialize
      output += sizeof(id->uid);

      *(int32_t *)output = id->tid;
      output += sizeof(id->tid);

      *(int32_t *)output = pQueryAttr->vgId;
      output += sizeof(pQueryAttr->vgId);

      char* data = NULL;
7651
      if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
7652 7653
        data = tsdbGetTableName(item->pTable);
      } else {
7654
        data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes);
7655 7656 7657 7658 7659 7660
      }

      doSetTagValueToResultBuf(output, data, type, bytes);
      count += 1;
    }

7661
    //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
7662 7663 7664 7665 7666 7667
  } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query
    SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0);
    *(int64_t*)pColInfo->pData = pInfo->totalTables;
    count = 1;

    pOperator->status = OP_EXEC_DONE;
7668
    //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count);
7669 7670 7671 7672 7673 7674 7675 7676 7677 7678 7679 7680 7681
  } else {  // return only the tags|table name etc.
    SExprInfo* pExprInfo = &pOperator->pExpr[0];  // todo use the column list instead of exprinfo

    count = 0;
    while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
      int32_t i = pInfo->curPos++;

      STableQueryInfo* item = taosArrayGetP(pa, i);

      char *data = NULL, *dst = NULL;
      int16_t type = 0, bytes = 0;
      for(int32_t j = 0; j < pOperator->numOfOutput; ++j) {
        // not assign value in case of user defined constant output column
7682
        if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) {
7683 7684 7685 7686 7687 7688 7689
          continue;
        }

        SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j);
        type  = pExprInfo[j].base.resSchema.type;
        bytes = pExprInfo[j].base.resSchema.bytes;

7690
        if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
7691 7692
          data = tsdbGetTableName(item->pTable);
        } else {
7693
          data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes);
7694 7695 7696 7697 7698 7699 7700 7701 7702 7703 7704 7705 7706
        }

        dst  = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes;
        doSetTagValueToResultBuf(dst, data, type, bytes);
      }

      count += 1;
    }

    if (pInfo->curPos >= pInfo->totalTables) {
      pOperator->status = OP_EXEC_DONE;
    }

7707
    //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
7708 7709 7710
  }

  if (pOperator->status == OP_EXEC_DONE) {
7711
    setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED);
7712 7713 7714 7715 7716 7717 7718 7719
  }

  pRes->info.rows = count;
  return (pRes->info.rows == 0)? NULL:pInfo->pRes;

#endif
}

H
Haojun Liao 已提交
7720
SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) {
7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731
  STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo));
  pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);

  size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv);
  assert(numOfGroup == 0 || numOfGroup == 1);

  pInfo->totalTables = pRuntimeEnv->tableqinfoGroupInfo.numOfTables;
  pInfo->curPos = 0;

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  pOperator->name         = "SeqTableTagScan";
X
Xiaoyu Wang 已提交
7732
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
7733 7734 7735 7736 7737 7738 7739
  pOperator->blockingOptr = false;
  pOperator->status       = OP_IN_EXECUTING;
  pOperator->info         = pInfo;
  pOperator->exec         = doTagScan;
  pOperator->pExpr        = pExpr;
  pOperator->numOfOutput  = numOfOutput;
  pOperator->pRuntimeEnv  = pRuntimeEnv;
H
Haojun Liao 已提交
7740
  pOperator->cleanupFn    = destroyTagScanOperatorInfo;
7741 7742 7743 7744 7745 7746 7747 7748 7749 7750 7751 7752 7753 7754 7755 7756 7757 7758 7759 7760 7761 7762 7763 7764 7765 7766 7767 7768 7769 7770 7771 7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803 7804

  return pOperator;
}
static bool initMultiDistinctInfo(SDistinctOperatorInfo *pInfo, SOperatorInfo* pOperator, SSDataBlock *pBlock) {
  if (taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput) {
     // distinct info already inited  
    return true;
  }
  for (int i = 0; i < pOperator->numOfOutput; i++) {
//    pInfo->totalBytes += pOperator->pExpr[i].base.colBytes;
  }
  for (int i = 0; i < pOperator->numOfOutput; i++) {
    int numOfBlock = (int)(taosArrayGetSize(pBlock->pDataBlock));
    assert(i < numOfBlock);
    for (int j = 0; j < numOfBlock; j++) {
      SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, j);
      if (pColDataInfo->info.colId == pOperator->pExpr[i].base.resSchema.colId) {
        SDistinctDataInfo item = {.index = j, .type = pColDataInfo->info.type, .bytes = pColDataInfo->info.bytes};
        taosArrayInsert(pInfo->pDistinctDataInfo, i, &item);
      }
    }
  }
  pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput);
  pInfo->buf        =  calloc(1, pInfo->totalBytes);
  return  taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false;
}

static void buildMultiDistinctKey(SDistinctOperatorInfo *pInfo, SSDataBlock *pBlock, int32_t rowId) {
  char *p = pInfo->buf;
  memset(p, 0, pInfo->totalBytes); 

  for (int i = 0; i < taosArrayGetSize(pInfo->pDistinctDataInfo); i++) {
    SDistinctDataInfo* pDistDataInfo = (SDistinctDataInfo *)taosArrayGet(pInfo->pDistinctDataInfo, i); 
    SColumnInfoData*   pColDataInfo = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index);
    char *val = ((char *)pColDataInfo->pData) + pColDataInfo->info.bytes * rowId;
    if (isNull(val, pDistDataInfo->type)) { 
      p += pDistDataInfo->bytes; 
      continue;
    }
    if (IS_VAR_DATA_TYPE(pDistDataInfo->type)) {
      memcpy(p, varDataVal(val), varDataLen(val));
      p += varDataLen(val);
    } else {
      memcpy(p, val, pDistDataInfo->bytes);
      p += pDistDataInfo->bytes;
    }
    memcpy(p, MULTI_KEY_DELIM, strlen(MULTI_KEY_DELIM));
    p += strlen(MULTI_KEY_DELIM);
  }
}

static SSDataBlock* hashDistinct(void* param, bool* newgroup) {
  SOperatorInfo* pOperator = (SOperatorInfo*) param;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SDistinctOperatorInfo* pInfo = pOperator->info;
  SSDataBlock* pRes = pInfo->pRes;

  pRes->info.rows = 0;
  SSDataBlock* pBlock = NULL;
   
  while(1) {
H
Haojun Liao 已提交
7805 7806 7807
    publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
    pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
    publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857

    if (pBlock == NULL) {
      doSetOperatorCompleted(pOperator);
      break;
    }
    if (!initMultiDistinctInfo(pInfo, pOperator, pBlock)) {
      doSetOperatorCompleted(pOperator);
      break;
    }
    // ensure result output buf 
    if (pRes->info.rows + pBlock->info.rows > pInfo->outputCapacity) {
      int32_t newSize = pRes->info.rows + pBlock->info.rows;
      for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) {
        SColumnInfoData*   pResultColInfoData = taosArrayGet(pRes->pDataBlock, i);
        SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo,  i);
        char* tmp = realloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes);
        if (tmp == NULL) {
          return NULL;
        } else {
          pResultColInfoData->pData = tmp;
        }
      }
      pInfo->outputCapacity = newSize;
    }

    for (int32_t i = 0; i < pBlock->info.rows; i++) {
      buildMultiDistinctKey(pInfo, pBlock, i);
      if (taosHashGet(pInfo->pSet, pInfo->buf, pInfo->totalBytes) == NULL) {
        int32_t dummy;
        taosHashPut(pInfo->pSet, pInfo->buf, pInfo->totalBytes, &dummy, sizeof(dummy));
        for (int j = 0; j < taosArrayGetSize(pRes->pDataBlock); j++) {
          SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, j);  // distinct meta info
          SColumnInfoData*   pColInfoData = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); //src
          SColumnInfoData*   pResultColInfoData = taosArrayGet(pRes->pDataBlock, j);  // dist 

          char* val = ((char*)pColInfoData->pData) + pDistDataInfo->bytes * i;
          char *start = pResultColInfoData->pData +  pDistDataInfo->bytes * pInfo->pRes->info.rows; 
          memcpy(start, val, pDistDataInfo->bytes);
        }
        pRes->info.rows += 1;
      } 
    }

    if (pRes->info.rows >= pInfo->threshold) {
      break;
    }
  }
  return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL;
}

H
Haojun Liao 已提交
7858
SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872
  SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo));
  pInfo->totalBytes      = 0;
  pInfo->buf             = NULL;
  pInfo->threshold       = tsMaxNumOfDistinctResults; // distinct result threshold
  pInfo->outputCapacity  = 4096;
  pInfo->pDistinctDataInfo = taosArrayInit(numOfOutput, sizeof(SDistinctDataInfo)); 
  pInfo->pSet = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
  pInfo->pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) pInfo->outputCapacity);
  

  SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
  pOperator->name         = "DistinctOperator";
  pOperator->blockingOptr = false;
  pOperator->status       = OP_IN_EXECUTING;
H
Haojun Liao 已提交
7873
//  pOperator->operatorType = OP_Distinct;
7874 7875 7876 7877 7878 7879
  pOperator->pExpr        = pExpr;
  pOperator->numOfOutput  = numOfOutput;
  pOperator->info         = pInfo;
  pOperator->pRuntimeEnv  = pRuntimeEnv;
  pOperator->exec         = hashDistinct;
  pOperator->pExpr        = pExpr; 
H
Haojun Liao 已提交
7880
  pOperator->cleanupFn    = destroyDistinctOperatorInfo;
7881

7882
    int32_t code = appendDownstream(pOperator, &downstream, 1);
7883 7884 7885 7886 7887 7888
  return pOperator;
}

static int32_t getColumnIndexInSource(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SColumnInfo* pTagCols) {
  int32_t j = 0;

7889 7890
  if (TSDB_COL_IS_TAG(pExpr->pColumns->flag)) {
    if (pExpr->pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
7891 7892 7893 7894
      return TSDB_TBNAME_COLUMN_INDEX;
    }

    while(j < pTableInfo->numOfTags) {
7895
      if (pExpr->pColumns->info.colId == pTagCols[j].colId) {
7896 7897 7898 7899 7900 7901
        return j;
      }

      j += 1;
    }

7902
  } /*else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) {  // user specified column data
7903 7904 7905 7906 7907 7908 7909 7910 7911
    return TSDB_UD_COLUMN_INDEX;
  } else {
    while (j < pTableInfo->numOfCols) {
      if (pExpr->colInfo.colId == pTableInfo->colList[j].colId) {
        return j;
      }

      j += 1;
    }
7912
  }*/
7913 7914 7915 7916 7917 7918 7919 7920 7921

  return INT32_MIN;  // return a less than TSDB_TBNAME_COLUMN_INDEX value
}

bool validateExprColumnInfo(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SColumnInfo* pTagCols) {
  int32_t j = getColumnIndexInSource(pTableInfo, pExpr, pTagCols);
  return j != INT32_MIN;
}

S
Shengliang Guan 已提交
7922
static bool validateQueryMsg(SQueryTableReq *pQueryMsg) {
7923 7924 7925 7926 7927
  if (pQueryMsg->interval.interval < 0) {
    //qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->interval.interval);
    return false;
  }

H
Haojun Liao 已提交
7928
//  if (pQueryMsg->sw.gap < 0 || pQueryMsg->sw.primaryColId != PRIMARYKEY_TIMESTAMP_COL_ID) {
7929
    //qError("qmsg:%p illegal value of session window time %" PRId64, pQueryMsg, pQueryMsg->sw.gap);
H
Haojun Liao 已提交
7930 7931
//    return false;
//  }
7932

H
Haojun Liao 已提交
7933
//  if (pQueryMsg->sw.gap > 0 && pQueryMsg->interval.interval > 0) {
7934 7935
    //qError("qmsg:%p illegal value of session window time %" PRId64" and interval value %"PRId64, pQueryMsg,
//        pQueryMsg->sw.gap, pQueryMsg->interval.interval);
H
Haojun Liao 已提交
7936 7937
//    return false;
//  }
7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963 7964 7965 7966 7967 7968 7969 7970 7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983 7984 7985 7986 7987

  if (pQueryMsg->numOfTables <= 0) {
    //qError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables);
    return false;
  }

  if (pQueryMsg->numOfGroupCols < 0) {
    //qError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryMsg, pQueryMsg->numOfGroupCols);
    return false;
  }

  if (pQueryMsg->numOfOutput > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutput <= 0) {
    //qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutput);
    return false;
  }

  return true;
}

static bool validateQueryTableCols(SQueriedTableInfo* pTableInfo, SSqlExpr** pExpr, int32_t numOfOutput,
                                   SColumnInfo* pTagCols, void* pMsg) {
  int32_t numOfTotal = pTableInfo->numOfCols + pTableInfo->numOfTags;
  if (pTableInfo->numOfCols < 0 || pTableInfo->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) {
    //qError("qmsg:%p illegal value of numOfCols %d numOfTags:%d", pMsg, pTableInfo->numOfCols, pTableInfo->numOfTags);
    return false;
  }

  if (numOfTotal == 0) {  // table total columns are not required.
//    for(int32_t i = 0; i < numOfOutput; ++i) {
//      SSqlExpr* p = pExpr[i];
//      if ((p->functionId == FUNCTION_TAGPRJ) ||
//          (p->functionId == FUNCTION_TID_TAG && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) ||
//          (p->functionId == FUNCTION_COUNT && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) ||
//          (p->functionId == FUNCTION_BLKINFO)) {
//        continue;
//      }
//
//      return false;
//    }
  }

  for(int32_t i = 0; i < numOfOutput; ++i) {
    if (!validateExprColumnInfo(pTableInfo, pExpr[i], pTagCols)) {
      return TSDB_CODE_QRY_INVALID_MSG;
    }
  }

  return true;
}

S
Shengliang Guan 已提交
7988
static char *createTableIdList(SQueryTableReq *pQueryMsg, char *pMsg, SArray **pTableIdList) {
7989 7990 7991 7992 7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009 8010 8011 8012 8013 8014 8015 8016 8017 8018 8019 8020 8021 8022 8023 8024 8025 8026 8027 8028 8029 8030 8031 8032 8033 8034 8035
  assert(pQueryMsg->numOfTables > 0);

  *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo));

  for (int32_t j = 0; j < pQueryMsg->numOfTables; ++j) {
    STableIdInfo* pTableIdInfo = (STableIdInfo *)pMsg;
    pTableIdInfo->uid = htobe64(pTableIdInfo->uid);
    pTableIdInfo->key = htobe64(pTableIdInfo->key);

    taosArrayPush(*pTableIdList, pTableIdInfo);
    pMsg += sizeof(STableIdInfo);
  }

  return pMsg;
}

static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) {
  for (int32_t f = 0; f < numOfFilters; ++f) {
    SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg);

    SColumnFilterInfo *pColFilter = &pColFilters[f];
    pColFilter->filterstr = htons(pFilterMsg->filterstr);

    (*pMsg) += sizeof(SColumnFilterInfo);

    if (pColFilter->filterstr) {
      pColFilter->len = htobe64(pFilterMsg->len);

      pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator
      if (pColFilter->pz == 0) {
        return TSDB_CODE_QRY_OUT_OF_MEMORY;
      }

      memcpy((void *)pColFilter->pz, (*pMsg), (size_t)pColFilter->len);
      (*pMsg) += (pColFilter->len + 1);
    } else {
      pColFilter->lowerBndi = htobe64(pFilterMsg->lowerBndi);
      pColFilter->upperBndi = htobe64(pFilterMsg->upperBndi);
    }

    pColFilter->lowerRelOptr = htons(pFilterMsg->lowerRelOptr);
    pColFilter->upperRelOptr = htons(pFilterMsg->upperRelOptr);
  }

  return TSDB_CODE_SUCCESS;
}

H
Haojun Liao 已提交
8036
static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) {
8037 8038
  SExecTaskInfo* pTaskInfo = calloc(1, sizeof(SExecTaskInfo));
  setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
H
Haojun Liao 已提交
8039

8040
  pTaskInfo->cost.created = taosGetTimestampMs();
H
Haojun Liao 已提交
8041
  pTaskInfo->id.queryId = queryId;
H
Haojun Liao 已提交
8042 8043

  char* p = calloc(1, 128);
H
Haojun Liao 已提交
8044
  snprintf(p, 128, "TID:0x%"PRIx64" QID:0x%"PRIx64, taskId, queryId);
8045
  pTaskInfo->id.str = strdup(p);
H
Haojun Liao 已提交
8046

8047 8048
  return pTaskInfo;
}
H
Haojun Liao 已提交
8049

X
Xiaoyu Wang 已提交
8050
static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId);
H
Haojun Liao 已提交
8051

H
Haojun Liao 已提交
8052
static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId);
H
Haojun Liao 已提交
8053

X
Xiaoyu Wang 已提交
8054
SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) {
H
Haojun Liao 已提交
8055 8056 8057 8058
  if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_PROJECT) { // ignore the project node
    pPhyNode = nodesListGetNode(pPhyNode->pChildren, 0);
  }

X
Xiaoyu Wang 已提交
8059 8060 8061
  if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) {
    if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pPhyNode)) {
      SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode;
H
Haojun Liao 已提交
8062

X
Xiaoyu Wang 已提交
8063 8064
      size_t numOfCols = LIST_LENGTH(pScanPhyNode->pScanCols);
      tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*) pPhyNode, pHandle, (uint64_t) queryId, taskId);
H
Haojun Liao 已提交
8065

H
Haojun Liao 已提交
8066
      int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId);
H
Haojun Liao 已提交
8067
      return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo);
X
Xiaoyu Wang 已提交
8068 8069 8070 8071 8072
    } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) {
      // SExchangePhysiNode* pEx = (SExchangePhysiNode*) pPhyNode;
      // return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo);
    } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == nodeType(pPhyNode)) {
      SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode;   // simple child table.
8073
      STableGroupInfo groupInfo = {0};
H
Haojun Liao 已提交
8074

H
Haojun Liao 已提交
8075
      int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId);
8076
      SArray* idList = NULL;
8077

8078 8079 8080
      if (groupInfo.numOfTables > 0) {
        SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0);
        ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1);
8081

8082 8083
        // Transfer the Array of STableKeyInfo into uid list.
        size_t  numOfTables = taosArrayGetSize(pa);
H
Haojun Liao 已提交
8084
        idList = taosArrayInit(numOfTables, sizeof(uint64_t));
H
Haojun Liao 已提交
8085

8086 8087 8088 8089 8090 8091
        for (int32_t i = 0; i < numOfTables; ++i) {
          STableKeyInfo* pkeyInfo = taosArrayGet(pa, i);
          taosArrayPush(idList, &pkeyInfo->uid);
        }
      } else {
        idList = taosArrayInit(4, sizeof(uint64_t));
H
Haojun Liao 已提交
8092 8093
      }

X
Xiaoyu Wang 已提交
8094 8095
      // SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pScanPhyNode->pScanCols, idList, pTaskInfo);
      // taosArrayDestroy(idList);
H
Haojun Liao 已提交
8096

X
Xiaoyu Wang 已提交
8097 8098
      // //TODO destroy groupInfo
      // return pOperator;
H
Haojun Liao 已提交
8099 8100 8101
    }
  }

X
Xiaoyu Wang 已提交
8102 8103
  if (QUERY_NODE_PHYSICAL_PLAN_AGG == nodeType(pPhyNode)) {
    size_t size = LIST_LENGTH(pPhyNode->pChildren);
H
Haojun Liao 已提交
8104 8105
    assert(size == 1);

H
Haojun Liao 已提交
8106 8107
    // TODO single table agg
    for (int32_t i = 0; i < size; ++i) {
X
Xiaoyu Wang 已提交
8108
      SPhysiNode*      pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i);
H
Haojun Liao 已提交
8109
      SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo);
X
Xiaoyu Wang 已提交
8110
      // return createAggregateOperatorInfo(op, pPhyNode->pTargets, pTaskInfo, pTableGroupInfo);
H
Haojun Liao 已提交
8111
    }
X
Xiaoyu Wang 已提交
8112
  } /*else if (pPhyNode->info.type == OP_MultiTableAggregate) {
H
Haojun Liao 已提交
8113 8114 8115
    size_t size = taosArrayGetSize(pPhyNode->pChildren);
    assert(size == 1);

H
Haojun Liao 已提交
8116
    for (int32_t i = 0; i < size; ++i) {
X
Xiaoyu Wang 已提交
8117
      SPhysiNode*      pChildNode = taosArrayGetP(pPhyNode->pChildren, i);
H
Haojun Liao 已提交
8118 8119
      SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo);
      return createMultiTableAggOperatorInfo(op, pPhyNode->pTargets, pTaskInfo, pTableGroupInfo);
8120
    }
X
Xiaoyu Wang 已提交
8121
  }*/
8122
}
H
Haojun Liao 已提交
8123

X
Xiaoyu Wang 已提交
8124
static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readHandle, uint64_t queryId, uint64_t taskId) {
H
Haojun Liao 已提交
8125
  STsdbQueryCond cond = {.loadExternalRows = false};
H
Haojun Liao 已提交
8126

H
Haojun Liao 已提交
8127
  cond.order = pTableScanNode->scan.order;
X
Xiaoyu Wang 已提交
8128
  cond.numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols);
H
Haojun Liao 已提交
8129
  cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo));
H
Haojun Liao 已提交
8130 8131 8132 8133 8134
  if (cond.colList == NULL) {
    terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
    return NULL;
  }

X
Xiaoyu Wang 已提交
8135
  cond.twindow = pTableScanNode->scanRange;
H
Haojun Liao 已提交
8136
  cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER;
H
Haojun Liao 已提交
8137
//  cond.type = pTableScanNode->scanFlag;
H
Haojun Liao 已提交
8138

H
Haojun Liao 已提交
8139
  for (int32_t i = 0; i < cond.numOfCols; ++i) {
H
Haojun Liao 已提交
8140
    STargetNode* pNode = (STargetNode*)nodesListGetNode(pTableScanNode->scan.pScanCols, i);
H
Haojun Liao 已提交
8141

H
Haojun Liao 已提交
8142 8143 8144 8145
    SColumnNode* pColNode = (SColumnNode*)pNode->pExpr;
    cond.colList[i].type = pColNode->colType;
    cond.colList[i].bytes = pColNode->node.resType.type;
    cond.colList[i].colId = pColNode->colId;
H
Haojun Liao 已提交
8146 8147
  }

H
Haojun Liao 已提交
8148
  return tsdbQueryTables(readHandle, &cond, pGroupInfo, queryId, taskId);
H
Haojun Liao 已提交
8149
}
H
Haojun Liao 已提交
8150

H
Haojun Liao 已提交
8151
static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) {
8152
  int32_t code = 0;
H
Haojun Liao 已提交
8153
  if (tableType == TSDB_SUPER_TABLE) {
H
Haojun Liao 已提交
8154
    code = tsdbQuerySTableByTagCond(metaHandle, tableUid, 0, NULL, 0, 0, NULL, pGroupInfo, NULL, 0, queryId, taskId);
H
Haojun Liao 已提交
8155
  } else {  // Create one table group.
H
Haojun Liao 已提交
8156
    code = tsdbGetOneTableGroup(metaHandle, tableUid, 0, pGroupInfo);
8157 8158 8159 8160
  }

  return code;
}
H
Haojun Liao 已提交
8161

X
Xiaoyu Wang 已提交
8162
static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId) {
8163
  STableGroupInfo groupInfo = {0};
H
Haojun Liao 已提交
8164

8165
  uint64_t uid = pTableScanNode->scan.uid;
H
Haojun Liao 已提交
8166
  int32_t code = doCreateTableGroup(pHandle->meta, pTableScanNode->scan.tableType, uid, &groupInfo, queryId, taskId);
8167 8168
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
H
Haojun Liao 已提交
8169
  }
H
Haojun Liao 已提交
8170

H
Haojun Liao 已提交
8171 8172
  if (groupInfo.numOfTables == 0) {
    code = 0;
H
Haojun Liao 已提交
8173
    qDebug("no table qualified for query, TID:0x%"PRIx64", QID:0x%"PRIx64, taskId, queryId);
H
Haojun Liao 已提交
8174 8175
    goto _error;
  }
H
Haojun Liao 已提交
8176

H
Haojun Liao 已提交
8177
  return createDataReaderImpl(pTableScanNode, &groupInfo, pHandle->reader, queryId, taskId);
H
Haojun Liao 已提交
8178

H
Haojun Liao 已提交
8179 8180 8181 8182 8183
  _error:
  terrno = code;
  return NULL;
}

H
Haojun Liao 已提交
8184
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId) {
H
Haojun Liao 已提交
8185 8186
  uint64_t queryId = pPlan->id.queryId;

H
Haojun Liao 已提交
8187
  int32_t code = TSDB_CODE_SUCCESS;
H
Haojun Liao 已提交
8188
  *pTaskInfo = createExecTaskInfo(queryId, taskId);
H
Haojun Liao 已提交
8189 8190 8191 8192
  if (*pTaskInfo == NULL) {
    code = TSDB_CODE_QRY_OUT_OF_MEMORY;
    goto _complete;
  }
H
Haojun Liao 已提交
8193

H
Haojun Liao 已提交
8194 8195
  STableGroupInfo group = {0};
  (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &group);
8196
  if ((*pTaskInfo)->pRoot == NULL) {
H
Haojun Liao 已提交
8197
    code = TSDB_CODE_QRY_OUT_OF_MEMORY;
H
Haojun Liao 已提交
8198
    goto _complete;
8199 8200
  }

H
Haojun Liao 已提交
8201 8202
  return code;

H
Haojun Liao 已提交
8203 8204 8205 8206 8207
_complete:
  tfree(*pTaskInfo);

  terrno = code;
  return code;
H
Haojun Liao 已提交
8208 8209
}

8210 8211 8212 8213 8214 8215 8216 8217 8218 8219 8220 8221 8222 8223 8224 8225 8226 8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237 8238 8239 8240 8241 8242 8243 8244 8245 8246 8247 8248 8249 8250 8251 8252 8253 8254 8255 8256 8257 8258 8259 8260 8261 8262 8263 8264 8265 8266 8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291
int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int32_t filterNum) {
  if (filterNum <= 0) {
    return TSDB_CODE_SUCCESS;
  }

  *dst = calloc(filterNum, sizeof(*src));
  if (*dst == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }

  memcpy(*dst, src, sizeof(*src) * filterNum);

  for (int32_t i = 0; i < filterNum; i++) {
    if ((*dst)[i].filterstr && dst[i]->len > 0) {
      void *pz = calloc(1, (size_t)(*dst)[i].len + 1);

      if (pz == NULL) {
        if (i == 0) {
          free(*dst);
        } else {
          freeColumnFilterInfo(*dst, i);
        }

        return TSDB_CODE_QRY_OUT_OF_MEMORY;
      }

      memcpy(pz, (void *)src->pz, (size_t)src->len + 1);

      (*dst)[i].pz = (int64_t)pz;
    }
  }

  return TSDB_CODE_SUCCESS;
}

int32_t buildArithmeticExprFromMsg(SExprInfo *pExprInfo, void *pQueryMsg) {
  //qDebug("qmsg:%p create arithmetic expr from binary", pQueryMsg);

  tExprNode* pExprNode = NULL;
  TRY(TSDB_MAX_TAG_CONDITIONS) {
    pExprNode = exprTreeFromBinary(pExprInfo->base.param[0].pz, pExprInfo->base.param[0].nLen);
  } CATCH( code ) {
    CLEANUP_EXECUTE();
    //qError("qmsg:%p failed to create arithmetic expression string from:%s, reason: %s", pQueryMsg, pExprInfo->base.param[0].pz, tstrerror(code));
    return code;
  } END_TRY

  if (pExprNode == NULL) {
    //qError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pExprInfo->base.param[0].pz);
    return TSDB_CODE_QRY_APP_ERROR;
  }

  pExprInfo->pExpr = pExprNode;
  return TSDB_CODE_SUCCESS;
}

static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SColumnInfo* pTagCols, SExprInfo* pExprs, int32_t numOfOutput, int32_t tagLen, bool superTable) {
  for (int32_t i = 0; i < numOfOutput; ++i) {
    int16_t functId = getExprFunctionId(&pExprs[i]);

    if (functId == FUNCTION_TOP || functId == FUNCTION_BOTTOM) {
      int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols);
      if (j < 0 || j >= pTableInfo->numOfCols) {
        return TSDB_CODE_QRY_INVALID_MSG;
      } else {
        SColumnInfo* pCol = &pTableInfo->colList[j];
//        int32_t ret = getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.param[0].i,
//                                        &pExprs[i].base.resSchema.type, &pExprs[i].base.resSchema.bytes, &pExprs[i].base.interBytes, tagLen, superTable, NULL);
//        assert(ret == TSDB_CODE_SUCCESS);
      }
    }
  }

  return TSDB_CODE_SUCCESS;
}

// TODO tag length should be passed from client, refactor
int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo,
                        SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo) {
  *pExprInfo = NULL;
  int32_t code = TSDB_CODE_SUCCESS;

8292
//  code = initUdfInfo(pUdfInfo);
8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308 8309 8310 8311 8312 8313 8314 8315 8316 8317 8318 8319 8320 8321 8322 8323 8324 8325 8326 8327 8328 8329 8330 8331
  if (code) {
    return code;
  }

  SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo));
  if (pExprs == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }

  bool    isSuperTable = /*QUERY_IS_STABLE_QUERY(queryType);*/ true;
  int16_t tagLen = 0;

  for (int32_t i = 0; i < numOfOutput; ++i) {
    pExprs[i].base = *pExprMsg[i];

    memset(pExprs[i].base.param, 0, sizeof(SVariant) * tListLen(pExprs[i].base.param));
    for (int32_t j = 0; j < pExprMsg[i]->numOfParams; ++j) {
      taosVariantAssign(&pExprs[i].base.param[j], &pExprMsg[i]->param[j]);
    }

    int16_t type = 0;
    int16_t bytes = 0;

    // parse the arithmetic expression
    int32_t functionId = getExprFunctionId(&pExprs[i]);
    if (functionId == FUNCTION_ARITHM) {
      code = buildArithmeticExprFromMsg(&pExprs[i], pMsg);

      if (code != TSDB_CODE_SUCCESS) {
        tfree(pExprs);
        return code;
      }

      type  = TSDB_DATA_TYPE_DOUBLE;
      bytes = tDataTypes[type].bytes;
    } else if (functionId == FUNCTION_BLKINFO) {
      SSchema s = {.type=TSDB_DATA_TYPE_BINARY, .bytes=TSDB_MAX_BINARY_LEN};
      type = s.type;
      bytes = s.bytes;
8332
    } else if (pExprs[i].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX && functionId == FUNCTION_TAGPRJ) {  // parse the normal column
H
Haojun Liao 已提交
8333
      const SSchema* s = tGetTbnameColumnSchema();
8334 8335
      type = s->type;
      bytes = s->bytes;
8336
    } else if (pExprs[i].base.pColumns->info.colId <= TSDB_UD_COLUMN_INDEX && pExprs[i].base.pColumns->info.colId > TSDB_RES_COL_ID) {
8337 8338 8339 8340 8341 8342 8343 8344 8345 8346
      // it is a user-defined constant value column
      assert(functionId == FUNCTION_PRJ);

      type = pExprs[i].base.param[1].nType;
      bytes = pExprs[i].base.param[1].nLen;
      if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
        bytes += VARSTR_HEADER_SIZE;
      }
    } else {
      int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols);
8347
      if (TSDB_COL_IS_TAG(pExprs[i].base.pColumns->flag)) {
8348 8349 8350 8351 8352 8353 8354 8355 8356 8357 8358
        if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pTableInfo->numOfTags) {
          tfree(pExprs);
          return TSDB_CODE_QRY_INVALID_MSG;
        }
      } else {
        if (j < PRIMARYKEY_TIMESTAMP_COL_ID || j >= pTableInfo->numOfCols) {
          tfree(pExprs);
          return TSDB_CODE_QRY_INVALID_MSG;
        }
      }

8359 8360
      if (pExprs[i].base.pColumns->info.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) {
        SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.pColumns->flag))? &pTagCols[j]:&pTableInfo->colList[j];
8361 8362 8363
        type = pCol->type;
        bytes = pCol->bytes;
      } else {
H
Haojun Liao 已提交
8364
        const SSchema* s = tGetTbnameColumnSchema();
8365 8366 8367 8368 8369 8370 8371 8372 8373 8374 8375 8376 8377 8378 8379 8380 8381 8382 8383 8384 8385 8386 8387 8388 8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399 8400 8401 8402 8403 8404 8405 8406 8407 8408 8409 8410 8411 8412 8413 8414 8415 8416 8417 8418 8419 8420 8421 8422 8423 8424 8425 8426 8427 8428

        type  = s->type;
        bytes = s->bytes;
      }

//      if (pExprs[i].base.flist.numOfFilters > 0) {
//        int32_t ret = cloneExprFilterInfo(&pExprs[i].base.flist.filterInfo, pExprMsg[i]->flist.filterInfo,
//            pExprMsg[i]->flist.numOfFilters);
//        if (ret) {
//          tfree(pExprs);
//          return ret;
//        }
//      }
    }

    int32_t param = (int32_t)pExprs[i].base.param[0].i;
//    if (functionId != FUNCTION_ARITHM &&
//       (type != pExprs[i].base.colType || bytes != pExprs[i].base.colBytes)) {
//      tfree(pExprs);
//      return TSDB_CODE_QRY_INVALID_MSG;
//    }

    // todo remove it
    SResultDataInfo info;
    if (getResultDataInfo(type, bytes, functionId, param, &info, 0, isSuperTable/*, pUdfInfo*/) != TSDB_CODE_SUCCESS) {
      tfree(pExprs);
      return TSDB_CODE_QRY_INVALID_MSG;
    }

    if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) {
      tagLen += pExprs[i].base.resSchema.bytes;
    }

    assert(isValidDataType(pExprs[i].base.resSchema.type));
  }

  // the tag length is affected by other tag columns, so this should be update.
  updateOutputBufForTopBotQuery(pTableInfo, pTagCols, pExprs, numOfOutput, tagLen, isSuperTable);

  *pExprInfo = pExprs;
  return TSDB_CODE_SUCCESS;
}

int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) {
  tExprNode* expr = NULL;
  
  TRY(TSDB_MAX_TAG_CONDITIONS) {
    expr = exprTreeFromBinary(data, len);
  } CATCH( code ) {
    CLEANUP_EXECUTE();
    return code;
  } END_TRY

  if (expr == NULL) {
    //qError("failed to create expr tree");
    return TSDB_CODE_QRY_APP_ERROR;
  }

//  int32_t ret = filterInitFromTree(expr, pFilters, 0);
//  tExprTreeDestroy(expr, NULL);

//  return ret;
}

S
Shengliang Guan 已提交
8429
SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableReq *pQueryMsg, SColIndex *pColIndex, int32_t *code) {
8430 8431 8432 8433 8434 8435 8436 8437 8438 8439 8440 8441 8442 8443 8444 8445 8446 8447 8448 8449 8450 8451 8452 8453 8454 8455 8456 8457 8458 8459 8460 8461 8462 8463 8464 8465 8466 8467 8468 8469 8470 8471 8472 8473 8474 8475 8476 8477 8478 8479 8480 8481 8482 8483 8484 8485 8486 8487 8488 8489 8490 8491 8492 8493 8494 8495 8496 8497 8498 8499 8500 8501 8502 8503 8504 8505 8506 8507 8508 8509 8510 8511 8512
  if (pQueryMsg->numOfGroupCols == 0) {
    return NULL;
  }

  // using group by tag columns
  SGroupbyExpr *pGroupbyExpr = (SGroupbyExpr *)calloc(1, sizeof(SGroupbyExpr));
  if (pGroupbyExpr == NULL) {
    *code = TSDB_CODE_QRY_OUT_OF_MEMORY;
    return NULL;
  }

  pGroupbyExpr->columnInfo = taosArrayInit(pQueryMsg->numOfGroupCols, sizeof(SColIndex));
  for(int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) {
    taosArrayPush(pGroupbyExpr->columnInfo, &pColIndex[i]);
  }

  return pGroupbyExpr;
}

//int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId) {
//  *pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols);
//  if (*pFilterInfo == NULL) {
//    return TSDB_CODE_QRY_OUT_OF_MEMORY;
//  }
//
//  for (int32_t i = 0, j = 0; i < numOfCols; ++i) {
//    if (pCols[i].flist.numOfFilters > 0) {
//      SSingleColumnFilterInfo* pFilter = &((*pFilterInfo)[j]);
//
//      memcpy(&pFilter->info, &pCols[i], sizeof(SColumnInfo));
//      pFilter->info = pCols[i];
//
//      pFilter->numOfFilters = pCols[i].flist.numOfFilters;
//      pFilter->pFilters = calloc(pFilter->numOfFilters, sizeof(SColumnFilterElem));
//      if (pFilter->pFilters == NULL) {
//        return TSDB_CODE_QRY_OUT_OF_MEMORY;
//      }
//
//      for (int32_t f = 0; f < pFilter->numOfFilters; ++f) {
//        SColumnFilterElem* pSingleColFilter = &pFilter->pFilters[f];
//        pSingleColFilter->filterInfo = pCols[i].flist.filterInfo[f];
//
//        int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr;
//        int32_t upper = pSingleColFilter->filterInfo.upperRelOptr;
//        if (lower == TSDB_RELATION_INVALID && upper == TSDB_RELATION_INVALID) {
//          //qError("QInfo:0x%"PRIx64" invalid filter info", qId);
//          return TSDB_CODE_QRY_INVALID_MSG;
//        }
//
//        pSingleColFilter->fp = getFilterOperator(lower, upper);
//        if (pSingleColFilter->fp == NULL) {
//          //qError("QInfo:0x%"PRIx64" invalid filter info", qId);
//          return TSDB_CODE_QRY_INVALID_MSG;
//        }
//
//        pSingleColFilter->bytes = pCols[i].bytes;
//
//        if (lower == TSDB_RELATION_IN) {
////          buildFilterSetFromBinary(&pSingleColFilter->q, (char *)(pSingleColFilter->filterInfo.pz), (int32_t)(pSingleColFilter->filterInfo.len));
//        }
//      }
//
//      j++;
//    }
//  }
//
//  return TSDB_CODE_SUCCESS;
//}

void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) {
//  for (int32_t i = 0; i < numOfFilterCols; ++i) {
//    if (pFilterInfo[i].numOfFilters > 0) {
//      if (pFilterInfo[i].pFilters->filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
//        taosHashCleanup((SHashObj *)(pFilterInfo[i].pFilters->q));
//      }
//      tfree(pFilterInfo[i].pFilters);
//    }
//  }
//
//  tfree(pFilterInfo);
  return NULL;
}

H
Haojun Liao 已提交
8513
int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId) {
8514 8515 8516 8517 8518 8519 8520 8521 8522 8523
  for (int32_t i = 0; i < pQueryAttr->numOfCols; ++i) {
//    if (pQueryAttr->tableCols[i].flist.numOfFilters > 0 && pQueryAttr->tableCols[i].flist.filterInfo != NULL) {
//      pQueryAttr->numOfFilterCols++;
//    }
  }

  if (pQueryAttr->numOfFilterCols == 0) {
    return TSDB_CODE_SUCCESS;
  }

8524 8525
//  doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols,
//                     &pQueryAttr->pFilterInfo, qId);
8526 8527 8528 8529 8530 8531

  pQueryAttr->createFilterOperator = true;

  return TSDB_CODE_SUCCESS;
}

H
Haojun Liao 已提交
8532
static void doUpdateExprColumnIndex(STaskAttr *pQueryAttr) {
8533 8534 8535 8536 8537 8538 8539 8540 8541
  assert(pQueryAttr->pExpr1 != NULL && pQueryAttr != NULL);

  for (int32_t k = 0; k < pQueryAttr->numOfOutput; ++k) {
    SSqlExpr *pSqlExprMsg = &pQueryAttr->pExpr1[k].base;
//    if (pSqlExprMsg->functionId == FUNCTION_ARITHM) {
//      continue;
//    }

    // todo opt performance
8542
    SColIndex *pColIndex = NULL;/*&pSqlExprMsg->colInfo;*/
8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556 8557 8558 8559 8560 8561 8562 8563 8564 8565 8566 8567 8568
    if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) {
      int32_t f = 0;
      for (f = 0; f < pQueryAttr->numOfCols; ++f) {
        if (pColIndex->colId == pQueryAttr->tableCols[f].colId) {
          pColIndex->colIndex = f;
          break;
        }
      }

      assert(f < pQueryAttr->numOfCols);
    } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) {
      // do nothing for user-defined constant value result columns
    } else {
      int32_t f = 0;
      for (f = 0; f < pQueryAttr->numOfTags; ++f) {
        if (pColIndex->colId == pQueryAttr->tagColList[f].colId) {
          pColIndex->colIndex = f;
          break;
        }
      }

      assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX);
    }
  }
}

H
Haojun Liao 已提交
8569
void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo) {
8570 8571 8572 8573 8574 8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586 8587 8588 8589 8590 8591 8592 8593 8594 8595 8596
  const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512);

  // the minimum number of rows for projection query
  const int32_t MIN_ROWS_FOR_PRJ_QUERY = 8192;
  const int32_t DEFAULT_MIN_ROWS = 4096;

  const float THRESHOLD_RATIO = 0.85f;

  if (isProjQuery(pQueryAttr)) {
    int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize;
    if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) {
      numOfRes = MIN_ROWS_FOR_PRJ_QUERY;
    }

    pResultInfo->capacity  = numOfRes;
  } else {  // in case of non-prj query, a smaller output buffer will be used.
    pResultInfo->capacity = DEFAULT_MIN_ROWS;
  }

  pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO);
  pResultInfo->total = 0;
}

FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) {
  return ((SQInfo *)qHandle)->qId == qId;
}

H
Haojun Liao 已提交
8597
int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start,
8598 8599 8600
                  int32_t prevResultLen, void* merger) {
  int32_t code = TSDB_CODE_SUCCESS;

H
Haojun Liao 已提交
8601
  STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
8602 8603
  pRuntimeEnv->qinfo = pQInfo;

H
Haojun Liao 已提交
8604
  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
8605 8606 8607 8608 8609 8610 8611 8612 8613 8614 8615 8616 8617 8618 8619 8620 8621 8622 8623 8624 8625 8626 8627 8628 8629 8630 8631 8632 8633 8634 8635 8636

  STSBuf *pTsBuf = NULL;

  if (pTsBufInfo->tsLen > 0) {  // open new file to save the result
    char* tsBlock = start + pTsBufInfo->tsOffset;
    pTsBuf = tsBufCreateFromCompBlocks(tsBlock, pTsBufInfo->tsNumOfBlocks, pTsBufInfo->tsLen, pTsBufInfo->tsOrder,
                                       pQueryAttr->vgId);

    if (pTsBuf == NULL) {
      code = TSDB_CODE_QRY_NO_DISKSPACE;
      goto _error;
    }
    tsBufResetPos(pTsBuf);
    bool ret = tsBufNextPos(pTsBuf);
    UNUSED(ret);
  }

  SArray* prevResult = NULL;
  if (prevResultLen > 0) {
    prevResult = interResFromBinary(param->prevResult, prevResultLen);
    pRuntimeEnv->prevResult = prevResult;
  }

  pRuntimeEnv->currentOffset = pQueryAttr->limit.offset;
  if (tsdb != NULL) {
//    pQueryAttr->precision = tsdbGetCfg(tsdb)->precision;
  }

  if ((QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.skey > pQueryAttr->window.ekey)) ||
      (!QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.ekey > pQueryAttr->window.skey))) {
    //qDebug("QInfo:0x%"PRIx64" no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo->qId, pQueryAttr->window.skey,
//           pQueryAttr->window.ekey, pQueryAttr->order.order);
H
Haojun Liao 已提交
8637
//    setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED);
8638 8639 8640 8641 8642 8643 8644
    pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0;
    // todo free memory
    return TSDB_CODE_SUCCESS;
  }

  if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) {
    //qDebug("QInfo:0x%"PRIx64" no table qualified for tag filter, abort query", pQInfo->qId);
H
Haojun Liao 已提交
8645
//    setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED);
8646 8647 8648 8649 8650 8651 8652 8653 8654 8655 8656 8657
    return TSDB_CODE_SUCCESS;
  }

  // filter the qualified
  if ((code = doInitQInfo(pQInfo, pTsBuf, tsdb, sourceOptr, param->tableScanOperator, param->pOperator, merger)) != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return code;

_error:
  // table query ref will be decrease during error handling
H
Haojun Liao 已提交
8658
//  doDestroyTask(pQInfo);
8659 8660 8661 8662 8663 8664 8665 8666 8667 8668 8669 8670 8671 8672 8673 8674 8675 8676 8677 8678 8679 8680 8681 8682 8683 8684 8685 8686 8687 8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711 8712 8713 8714 8715 8716 8717 8718 8719 8720 8721 8722 8723 8724 8725 8726 8727 8728 8729 8730 8731 8732 8733 8734 8735 8736 8737 8738
  return code;
}

//TODO refactor
void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) {
    if (pFilter == NULL || numOfFilters == 0) {
      return;
    }

    for (int32_t i = 0; i < numOfFilters; i++) {
      if (pFilter[i].filterstr && pFilter[i].pz) {
        free((void*)(pFilter[i].pz));
      }
    }

    free(pFilter);
}

static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo) {
  if (pTableqinfoGroupInfo->pGroupList != NULL) {
    int32_t numOfGroups = (int32_t) taosArrayGetSize(pTableqinfoGroupInfo->pGroupList);
    for (int32_t i = 0; i < numOfGroups; ++i) {
      SArray *p = taosArrayGetP(pTableqinfoGroupInfo->pGroupList, i);

      size_t num = taosArrayGetSize(p);
      for(int32_t j = 0; j < num; ++j) {
        STableQueryInfo* item = taosArrayGetP(p, j);
        destroyTableQueryInfoImpl(item);
      }

      taosArrayDestroy(p);
    }
  }

  taosArrayDestroy(pTableqinfoGroupInfo->pGroupList);
  taosHashCleanup(pTableqinfoGroupInfo->map);

  pTableqinfoGroupInfo->pGroupList = NULL;
  pTableqinfoGroupInfo->map = NULL;
  pTableqinfoGroupInfo->numOfTables = 0;
}

void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) {
  if (pExprInfo == NULL) {
    assert(numOfExpr == 0);
    return NULL;
  }

  for (int32_t i = 0; i < numOfExpr; ++i) {
    if (pExprInfo[i].pExpr != NULL) {
      tExprTreeDestroy(pExprInfo[i].pExpr, NULL);
    }

//    if (pExprInfo[i].base.flist.filterInfo) {
//      freeColumnFilterInfo(pExprInfo[i].base.flist.filterInfo, pExprInfo[i].base.flist.numOfFilters);
//    }

    for(int32_t j = 0; j < pExprInfo[i].base.numOfParams; ++j) {
      taosVariantDestroy(&pExprInfo[i].base.param[j]);
    }
  }

  tfree(pExprInfo);
  return NULL;
}

void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) {
  if (pColumnInfo != NULL) {
    assert(numOfCols >= 0);

    for (int32_t i = 0; i < numOfCols; i++) {
      freeColumnFilterInfo(pColumnInfo[i].flist.filterInfo, pColumnInfo[i].flist.numOfFilters);
    }

    tfree(pColumnInfo);
  }

  return NULL;
}

H
Haojun Liao 已提交
8739
void doDestroyTask(SExecTaskInfo *pTaskInfo) {
H
Haojun Liao 已提交
8740 8741
  qDebug("%s execTask is freed", GET_TASKID(pTaskInfo));

H
Haojun Liao 已提交
8742 8743 8744
  doDestroyTableQueryInfo(&pTaskInfo->tableqinfoGroupInfo);
//  taosArrayDestroy(pTaskInfo->summary.queryProfEvents);
//  taosHashCleanup(pTaskInfo->summary.operatorProfResults);
8745

8746 8747 8748
  tfree(pTaskInfo->sql);
  tfree(pTaskInfo->id.str);
  tfree(pTaskInfo);
8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780 8781 8782 8783 8784 8785 8786 8787 8788 8789 8790 8791 8792 8793 8794 8795 8796 8797 8798 8799 8800 8801 8802 8803
}

static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) {
  if (val == NULL) {
    setNull(output, type, bytes);
    return;
  }

  if (IS_VAR_DATA_TYPE(type)) {
    // Binary data overflows for sort of unknown reasons. Let trim the overflow data
    if (varDataTLen(val) > bytes) {
      int32_t maxLen = bytes - VARSTR_HEADER_SIZE;
      int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val);
      memcpy(varDataVal(output), varDataVal(val), len);
      varDataSetLen(output, len);
    } else {
      varDataCopy(output, val);
    }
  } else {
    memcpy(output, val, bytes);
  }
}

static int64_t getQuerySupportBufSize(size_t numOfTables) {
  size_t s1 = sizeof(STableQueryInfo);
  size_t s2 = sizeof(SHashNode);

//  size_t s3 = sizeof(STableCheckInfo);  buffer consumption in tsdb
  return (int64_t)((s1 + s2) * 1.5 * numOfTables);
}

int32_t checkForQueryBuf(size_t numOfTables) {
  int64_t t = getQuerySupportBufSize(numOfTables);
  if (tsQueryBufferSizeBytes < 0) {
    return TSDB_CODE_SUCCESS;
  } else if (tsQueryBufferSizeBytes > 0) {

    while(1) {
      int64_t s = tsQueryBufferSizeBytes;
      int64_t remain = s - t;
      if (remain >= 0) {
        if (atomic_val_compare_exchange_64(&tsQueryBufferSizeBytes, s, remain) == s) {
          return TSDB_CODE_SUCCESS;
        }
      } else {
        return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER;
      }
    }
  }

  // disable query processing if the value of tsQueryBufferSize is zero.
  return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER;
}

bool checkNeedToCompressQueryCol(SQInfo *pQInfo) {
H
Haojun Liao 已提交
8804 8805
  STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
  STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
8806 8807 8808 8809 8810 8811 8812 8813 8814 8815 8816 8817 8818 8819 8820 8821 8822 8823 8824 8825 8826 8827 8828 8829 8830 8831 8832 8833 8834 8835 8836

  SSDataBlock* pRes = pRuntimeEnv->outputBuf;

  if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) <= 0) {
    return false;
  }

  int32_t numOfRows = pQueryAttr->pExpr2 ? GET_NUM_OF_RESULTS(pRuntimeEnv) : pRes->info.rows;
  int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput;

  for (int32_t col = 0; col < numOfCols; ++col) {
    SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col);
    int32_t colSize = pColRes->info.bytes * numOfRows;
    if (NEEDTO_COMPRESS_QUERY(colSize)) {
      return true;
    }
  }

  return false;
}

void releaseQueryBuf(size_t numOfTables) {
  if (tsQueryBufferSizeBytes < 0) {
    return;
  }

  int64_t t = getQuerySupportBufSize(numOfTables);

  // restore value is not enough buffer available
  atomic_add_fetch_64(&tsQueryBufferSizeBytes, t);
}