timewindowoperator.c 194.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15
#include "executorimpl.h"
X
Xiaoyu Wang 已提交
16
#include "function.h"
5
54liuyao 已提交
17
#include "functionMgt.h"
5
54liuyao 已提交
18
#include "tcompare.h"
L
Liu Jicong 已提交
19
#include "tdatablock.h"
H
Haojun Liao 已提交
20
#include "tfill.h"
21
#include "ttime.h"
22 23 24 25 26 27

typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;

5
54liuyao 已提交
28 29
#define IS_FINAL_OP(op) ((op)->isFinal)

5
54liuyao 已提交
30 31 32 33 34 35 36
typedef struct SWinRes {
  TSKEY    ts;
  uint64_t groupId;
} SWinRes;

typedef struct SPullWindowInfo {
  STimeWindow window;
L
Liu Jicong 已提交
37
  uint64_t    groupId;
5
54liuyao 已提交
38 39
} SPullWindowInfo;

5
54liuyao 已提交
40
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
5
54liuyao 已提交
41

42 43 44 45 46
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);

static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult);
static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult);

H
Haojun Liao 已提交
47 48 49 50 51 52 53 54 55
///*
// * There are two cases to handle:
// *
// * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including
// * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey.
// * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be
// *    merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there
// *    is a previous result generated or not.
// */
56
// static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) {
H
Haojun Liao 已提交
57 58
//  // do nothing
//}
59

X
Xiaoyu Wang 已提交
60
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }
61 62 63

static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
64
                                      int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
65 66 67 68 69 70 71 72 73 74 75
                                      SExecTaskInfo* pTaskInfo) {
  SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                  masterscan, tableGroupId, pTaskInfo, true, pAggSup);

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // set time window for current result
  pResultRow->win = (*win);
76

77
  *pResult = pResultRow;
78
  setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
79

80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
  return TSDB_CODE_SUCCESS;
}

static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) {
  int64_t* ts = (int64_t*)pColData->pData;
  int32_t  delta = includeEndpoint ? 1 : 0;

  int64_t duration = pWin->ekey - pWin->skey + delta;
  ts[2] = duration;            // set the duration
  ts[3] = pWin->skey;          // window start key
  ts[4] = pWin->ekey + delta;  // window end key
}

static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) {
  pRowSup->win.ekey = ts;
  pRowSup->prevTs = ts;
  pRowSup->numOfRows += 1;
}

static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex) {
  pRowSup->startRowIndex = rowIndex;
  pRowSup->numOfRows = 0;
  pRowSup->win.skey = tsList[rowIndex];
}

static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
                                                   int16_t pos, int16_t order, int64_t* pData) {
107
  int32_t forwardRows = 0;
108 109 110 111

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
112
      forwardRows = end;
113 114

      if (pData[end + pos] == ekey) {
115
        forwardRows += 1;
116 117 118
      }
    }
  } else {
119
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
120
    if (end >= 0) {
121
      forwardRows = end;
122

123
      if (pData[end + pos] == ekey) {
124
        forwardRows += 1;
125 126
      }
    }
X
Xiaoyu Wang 已提交
127 128 129 130 131 132 133 134
    //    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
    //    if (end >= 0) {
    //      forwardRows = pos - end;
    //
    //      if (pData[end] == ekey) {
    //        forwardRows += 1;
    //      }
    //    }
135 136
  }

137 138
  assert(forwardRows >= 0);
  return forwardRows;
139 140
}

5
54liuyao 已提交
141
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY*  keyList = (TSKEY*)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
158 159 160 161 162 163 164 165 166 167 168
      if (key >= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key < keyList[lastPos]) {
        lastPos += 1;
        if (lastPos >= num) {
          return -1;
        } else {
          return lastPos;
        }
      }
169 170 171 172 173 174

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        firstPos = midPos + 1;
175 176
      } else if (key > keyList[midPos]) {
        lastPos = midPos - 1;
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

X
Xiaoyu Wang 已提交
212 213
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
                                 __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num = -1;
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);

  if (order == TSDB_ORDER_ASC) {
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
      if (item != NULL) {
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
235
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
236 237
      }
    } else {
238
      num = pDataBlockInfo->rows - startPos;
239
      if (item != NULL) {
240
        item->lastKey = pDataBlockInfo->window.ekey + step;
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
      }
    }
  }

  assert(num >= 0);
  return num;
}

static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
    return;
  }

  int64_t key = tw->skey, interval = pInterval->interval;
  // convert key to second
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;

  if (pInterval->intervalUnit == 'y') {
    interval *= 12;
  }

  struct tm tm;
  time_t    t = (time_t)key;
  taosLocalTime(&t, &tm);

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
272
  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
273 274 275 276

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
277
  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
278 279 280 281

  tw->ekey -= 1;
}

282 283
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs,
                               int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
284
  SqlFunctionCtx* pCtx = pSup->pCtx;
285

286
  int32_t index = 1;
287
  for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
H
Haojun Liao 已提交
288
    if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
289 290 291 292
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

X
Xiaoyu Wang 已提交
293
    SFunctParam*     pParam = &pCtx[k].param[0];
294 295
    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId);

296
    ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey);
297

298
    double v1 = 0, v2 = 0, v = 0;
299
    if (prevRowIndex == -1) {
300
      SGroupKeys* p = taosArrayGet(pPrevValues, index);
301
      GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData);
302
    } else {
303
      GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex));
304 305
    }

306
    GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
307

308
#if 0
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            //            pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
328 329
#endif

X
Xiaoyu Wang 已提交
330 331 332
    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
    SPoint point = (SPoint){.key = windowKey, .val = &v};
333

X
Xiaoyu Wang 已提交
334
    taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
335

X
Xiaoyu Wang 已提交
336 337 338 339 340 341
    if (type == RESULT_ROW_START_INTERP) {
      pCtx[k].start.key = point.key;
      pCtx[k].start.val = v;
    } else {
      pCtx[k].end.key = point.key;
      pCtx[k].end.val = v;
342
    }
X
Xiaoyu Wang 已提交
343 344 345

    index += 1;
  }
346
#if 0
347
  }
348
#endif
349 350 351 352 353 354 355 356 357 358 359 360 361 362
}

static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

363 364
static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock,
                                              const TSKEY* tsCols, STimeWindow* win, SExprSupp* pSup) {
X
Xiaoyu Wang 已提交
365
  bool ascQuery = (pInfo->order == TSDB_ORDER_ASC);
366

367
  TSKEY curTs = tsCols[pos];
368 369

  SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
X
Xiaoyu Wang 已提交
370
  TSKEY       lastTs = *(int64_t*)pTsKey->pData;
371 372 373 374 375

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery ? win->skey : win->ekey;
  if (key == curTs) {
376
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
377 378 379
    return true;
  }

380 381
  // it is the first time window, no need to do interpolation
  if (pTsKey->isNull && pos == 0) {
382
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
383 384
  } else {
    TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]);
385 386
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key,
                              RESULT_ROW_START_INTERP, pSup);
387 388 389 390 391
  }

  return true;
}

392 393 394
static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex,
                                            SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey,
                                            STimeWindow* win) {
395
  int32_t order = pInfo->order;
396 397

  TSKEY actualEndKey = tsCols[endRowIndex];
398
  TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey;
399 400

  // not ended in current data block, do not invoke interpolation
401
  if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) {
402
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
403 404 405
    return false;
  }

406
  // there is actual end point of current time window, no interpolation needs
407
  if (key == actualEndKey) {
408
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
409 410 411
    return true;
  }

412
  int32_t nextRowIndex = endRowIndex + 1;
413 414 415
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
416 417
  doTimeWindowInterpolation(pInfo->pPrevValues, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key,
                            RESULT_ROW_END_INTERP, pSup);
418 419 420
  return true;
}

5
54liuyao 已提交
421
bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo) {
422 423
  if (pInterval->interval != pInterval->sliding &&
      (pWin->ekey < pBlockInfo->calWin.skey || pWin->skey > pBlockInfo->calWin.ekey)) {
5
54liuyao 已提交
424 425 426 427 428
    return false;
  }
  return true;
}

429
static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
5
54liuyao 已提交
430
                                      TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
X
Xiaoyu Wang 已提交
431
  bool ascQuery = (order == TSDB_ORDER_ASC);
432 433 434 435 436 437 438 439 440 441

  int32_t precision = pInterval->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
    return -1;
  }

5
54liuyao 已提交
442 443 444 445
  if (!inSlidingWindow(pInterval, pNext, pDataBlockInfo) && order == TSDB_ORDER_ASC) {
    return -1;
  }

446
  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
447 448 449 450
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
451
    startPos = prevPosition + 1;
452
  } else {
453
    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
454 455
      startPos = 0;
    } else {
456
      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
    }
  }

  /* interp query with fill should not skip time window */
  //  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
  //    return startPos;
  //  }

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
    if (ascQuery) {
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
      }
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
      }
    }
  }

  return startPos;
}

500 501
static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
  ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp == true;
  }
}

static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp = true;
  }
}

518 519
static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult,
                                        STimeWindow* win, int32_t startPos, int32_t forwardRows, SExprSupp* pSup) {
520
  if (!pInfo->timeWindowInterpo) {
521 522 523
    return;
  }

524
  ASSERT(pBlock != NULL);
525 526 527 528 529
  if (pBlock->pDataBlock == NULL) {
    //    tscError("pBlock->pDataBlock == NULL");
    return;
  }

530
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
531 532

  TSKEY* tsCols = (TSKEY*)(pColInfo->pData);
533
  bool   done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
534
  if (!done) {  // it is not interpolated, now start to generated the interpolated value
535
    bool interp = setTimeWindowInterpolationStartTs(pInfo, startPos, pBlock, tsCols, win, pSup);
536 537 538 539
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
540
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
541 542 543 544 545 546 547 548
  }

  // point interpolation does not require the end key time window interpolation.
  //  if (pointInterpQuery) {
  //    return;
  //  }

  // interpolation query does not generate the time window end interpolation
549
  done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
550
  if (!done) {
551
    int32_t endRowIndex = startPos + forwardRows - 1;
552

553
    TSKEY endKey = (pInfo->order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
554
    bool  interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
555 556 557 558
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
559
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
560 561 562
  }
}

563 564
static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) {
  if (pBlock->pDataBlock == NULL) {
565 566 567
    return;
  }

568 569 570 571 572 573 574
  size_t num = taosArrayGetSize(pPrevKeys);
  for (int32_t k = 0; k < num; ++k) {
    SColumn* pc = taosArrayGet(pCols, k);

    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId);

    SGroupKeys* pkey = taosArrayGet(pPrevKeys, k);
X
Xiaoyu Wang 已提交
575
    for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) {
576 577 578 579 580 581 582 583 584 585 586 587 588 589
      if (colDataIsNull_s(pColInfo, i)) {
        continue;
      }

      char* val = colDataGetData(pColInfo, i);
      if (IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, varDataTLen(val));
        ASSERT(varDataTLen(val) <= pkey->bytes);
      } else {
        memcpy(pkey->pData, val, pkey->bytes);
      }

      break;
    }
590 591 592
  }
}

593 594 595 596
static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo,
                                       SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) {
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

597
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
598
  SExprSupp*                pSup = &pOperatorInfo->exprSupp;
599

600
  int32_t  startPos = 0;
601
  int32_t  numOfOutput = pSup->numOfExprs;
602
  uint64_t groupId = pBlock->info.groupId;
603

604
  SResultRow* pResult = NULL;
605

606 607
  while (1) {
    SListNode* pn = tdListGetHead(pResultRowInfo->openWindow);
608

609 610 611 612
    SResultRowPosition* p1 = (SResultRowPosition*)pn->data;
    if (p->pageId == p1->pageId && p->offset == p1->offset) {
      break;
    }
613

614 615
    SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1);
    ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
616

617
    if (pr->closed) {
X
Xiaoyu Wang 已提交
618 619
      ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
             isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
620 621 622
      tdListPopHead(pResultRowInfo->openWindow);
      continue;
    }
623

624
    STimeWindow w = pr->win;
625 626
    int32_t     ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx,
                                             numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
627 628 629 630 631 632
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP));

X
Xiaoyu Wang 已提交
633 634
    SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
    int64_t     prevTs = *(int64_t*)pTsKey->pData;
635 636
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey,
                              RESULT_ROW_END_INTERP, pSup);
637 638

    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
639
    setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
640

641 642
    doApplyFunctions(pTaskInfo, pSup->pCtx, &w, &pInfo->twAggSup.timeWindowData, startPos, 0, tsCols, pBlock->info.rows,
                     numOfExprs, pInfo->order);
643 644 645 646

    if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
      closeResultRow(pr);
      tdListPopHead(pResultRowInfo->openWindow);
X
Xiaoyu Wang 已提交
647
    } else {  // the remains are can not be closed yet.
648
      break;
649
    }
650
  }
651
}
652

5
54liuyao 已提交
653
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
5
54liuyao 已提交
654
  if (!pBlock || pBlock->info.rows == 0) {
5
54liuyao 已提交
655
    qDebug("===stream===printDataBlock: Block is Null or Empty");
5
54liuyao 已提交
656 657
    return;
  }
L
Liu Jicong 已提交
658
  char* pBuf = NULL;
5
54liuyao 已提交
659 660
  qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
  taosMemoryFree(pBuf);
5
54liuyao 已提交
661 662
}

5
54liuyao 已提交
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);

int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;
      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;
      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;
      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

      if (comparefn(pKey, keyList, lastPos) > 0) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

5
54liuyao 已提交
719
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
720

X
Xiaoyu Wang 已提交
721 722 723
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;
5
54liuyao 已提交
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (key >= getValuefn(keyList, lastPos)) return lastPos;
      if (key == getValuefn(keyList, firstPos)) return firstPos;
      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (key <= getValuefn(keyList, firstPos)) return firstPos;
      if (key == getValuefn(keyList, lastPos)) return lastPos;

      if (key > getValuefn(keyList, lastPos)) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
770 771
  }

5
54liuyao 已提交
772 773 774
  return midPos;
}

5
54liuyao 已提交
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
int32_t compareResKey(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
  SResKeyPos* pos = taosArrayGetP(res, index);
  SWinRes*    pData = (SWinRes*) pKey;
  if (pData->ts == *(int64_t*)pos->key) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->ts > *(int64_t*)pos->key) {
    return 1;
  }
  return -1;
}

792
static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SArray* pUpdated) {
5
54liuyao 已提交
793
  int32_t size = taosArrayGetSize(pUpdated);
5
54liuyao 已提交
794 795
  SWinRes data = {.ts = ts, .groupId = groupId};
  int32_t index = binarySearchCom(pUpdated, size, &data, TSDB_ORDER_DESC, compareResKey);
5
54liuyao 已提交
796 797 798
  if (index == -1) {
    index = 0;
  } else {
5
54liuyao 已提交
799
    if (compareResKey(&data, pUpdated, index) > 0) {
5
54liuyao 已提交
800 801 802 803 804
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
H
Haojun Liao 已提交
805

5
54liuyao 已提交
806 807 808 809 810
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
5
54liuyao 已提交
811 812
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
X
Xiaoyu Wang 已提交
813
  if (taosArrayInsert(pUpdated, index, &newPos) == NULL) {
5
54liuyao 已提交
814 815 816 817 818
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
819 820 821 822
static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
  return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated);
}

5
54liuyao 已提交
823
static void removeResult(SArray* pUpdated, SWinRes* pKey) {
5
54liuyao 已提交
824
  int32_t size = taosArrayGetSize(pUpdated);
5
54liuyao 已提交
825 826
  int32_t index = binarySearchCom(pUpdated, size, pKey, TSDB_ORDER_DESC, compareResKey);
  if (index >= 0 && 0 == compareResKey(pKey, pUpdated, index)) {
5
54liuyao 已提交
827 828 829 830 831 832 833
    taosArrayRemove(pUpdated, index);
  }
}

static void removeResults(SArray* pWins, SArray* pUpdated) {
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
834
    SWinRes* pW = taosArrayGet(pWins, i);
5
54liuyao 已提交
835
    removeResult(pUpdated, pW);
5
54liuyao 已提交
836 837 838
  }
}

839
int64_t getWinReskey(void* data, int32_t index) {
840
  SArray*  res = (SArray*)data;
841 842 843 844
  SWinRes* pos = taosArrayGet(res, index);
  return pos->ts;
}

5
54liuyao 已提交
845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
int32_t compareWinRes(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
  SWinRes*    pos = taosArrayGetP(res, index);
  SResKeyPos* pData = (SResKeyPos*) pKey;
  if (*(int64_t*)pData->key == pos->ts) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (*(int64_t*)pData->key > pos->ts) {
    return 1;
  }
  return -1;
}

862 863 864 865 866
static void removeDeleteResults(SArray* pUpdated, SArray* pDelWins) {
  int32_t upSize = taosArrayGetSize(pUpdated);
  int32_t delSize = taosArrayGetSize(pDelWins);
  for (int32_t i = 0; i < upSize; i++) {
    SResKeyPos* pResKey = taosArrayGetP(pUpdated, i);
5
54liuyao 已提交
867 868
    int32_t     index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes);
    if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) {
869 870 871 872 873
      taosArrayRemove(pDelWins, index);
    }
  }
}

5
54liuyao 已提交
874 875 876 877 878
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) {
  ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
  return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark;
}

L
Liu Jicong 已提交
879
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); }
5
54liuyao 已提交
880

5
54liuyao 已提交
881
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
882
                            int32_t scanFlag, SArray* pUpdated) {
883
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
884

885
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
886
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
887

X
Xiaoyu Wang 已提交
888
  int32_t     startPos = 0;
889
  int32_t     numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
890 891 892 893 894
  int64_t*    tsCols = extractTsCol(pBlock, pInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
  bool        ascScan = (pInfo->order == TSDB_ORDER_ASC);
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;
895

896
  STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order);
L
Liu Jicong 已提交
897
  int32_t     ret = TSDB_CODE_SUCCESS;
898 899
  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
L
Liu Jicong 已提交
900 901
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
902 903 904
    if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
905

L
Liu Jicong 已提交
906
    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
907
      saveResultRow(pResult, tableGroupId, pUpdated);
5
54liuyao 已提交
908
    }
909 910
  }

X
Xiaoyu Wang 已提交
911 912 913
  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
914
  ASSERT(forwardRows > 0);
915 916

  // prev time window not interpolation yet.
917
  if (pInfo->timeWindowInterpo) {
918 919
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult);
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);
920 921

    // restore current time window
922 923
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
924 925 926 927
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

928
    // window start key interpolation
929
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
930
  }
931

932 933
  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
5
54liuyao 已提交
934 935 936 937
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
                     pBlock->info.rows, numOfOutput, pInfo->order);
  }
938 939

  doCloseWindow(pResultRowInfo, pInfo, pResult);
940 941 942

  STimeWindow nextWin = win;
  while (1) {
943
    int32_t prevEndPos = forwardRows - 1 + startPos;
944
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->order);
945 946 947
    if (startPos < 0) {
      break;
    }
5
54liuyao 已提交
948
    if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
5
54liuyao 已提交
949 950 951 952 953
      ekey = ascScan ? nextWin.ekey : nextWin.skey;
      forwardRows =
          getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
      continue;
    }
954 955

    // null data, failed to allocate more memory buffer
X
Xiaoyu Wang 已提交
956
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
957
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
958 959 960 961
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

L
Liu Jicong 已提交
962
    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
963
      saveResultRow(pResult, tableGroupId, pUpdated);
964 965
    }

X
Xiaoyu Wang 已提交
966
    ekey = ascScan ? nextWin.ekey : nextWin.skey;
967
    forwardRows =
968
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
969 970

    // window start(end) key interpolation
971
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
972 973

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
974 975
    doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
                     pBlock->info.rows, numOfOutput, pInfo->order);
976
    doCloseWindow(pResultRowInfo, pInfo, pResult);
977 978 979
  }

  if (pInfo->timeWindowInterpo) {
980
    saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
981
  }
982 983 984 985 986 987 988 989 990 991 992 993
}

void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
  // current result is done in computing final results.
  if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
    closeResultRow(pResult);
    tdListPopHead(pResultRowInfo->openWindow);
  }
}

SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult) {
  SResultRowPosition pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
X
Xiaoyu Wang 已提交
994
  SListNode*         pn = tdListGetTail(pResultRowInfo->openWindow);
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
  if (pn == NULL) {
    tdListAppend(pResultRowInfo->openWindow, &pos);
    return pos;
  }

  SResultRowPosition* px = (SResultRowPosition*)pn->data;
  if (px->pageId != pos.pageId || px->offset != pos.offset) {
    tdListAppend(pResultRowInfo->openWindow, &pos);
  }

  return pos;
}

int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
  TSKEY* tsCols = NULL;
1010

1011 1012 1013 1014
  if (pBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;

1015 1016 1017 1018 1019 1020
    // no data in primary ts
    if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) {
      return NULL;
    }

    if (tsCols[0] != 0 && (pBlock->info.window.skey == 0 && pBlock->info.window.ekey == 0)) {
1021 1022 1023 1024 1025
      blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
    }
  }

  return tsCols;
1026 1027 1028 1029 1030 1031 1032
}

static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
  }

L
Liu Jicong 已提交
1033
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1034
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
1035
  SExprSupp*                pSup = &pOperator->exprSupp;
1036

1037 1038
  int32_t scanFlag = MAIN_SCAN;

X
Xiaoyu Wang 已提交
1039
  int64_t        st = taosGetTimestampUs();
1040 1041 1042
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1043
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1044 1045 1046 1047
    if (pBlock == NULL) {
      break;
    }

1048 1049
    getTableScanInfo(pOperator, &pInfo->order, &scanFlag);

1050
    if (pInfo->scalarSupp.pExprInfo != NULL) {
L
Liu Jicong 已提交
1051 1052
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
1053 1054
    }

1055
    // the pDataBlock are always the same one, no need to call this again
1056
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, scanFlag, true);
H
Haojun Liao 已提交
1057
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL);
1058 1059 1060
  }

  closeAllResultRows(&pInfo->binfo.resultRowInfo);
1061
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->order);
1062
  OPTR_SET_OPENED(pOperator);
1063 1064

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1065 1066 1067
  return TSDB_CODE_SUCCESS;
}

1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
static bool compareVal(const char* v, const SStateKeys* pKey) {
  if (IS_VAR_DATA_TYPE(pKey->type)) {
    if (varDataLen(v) != varDataLen(pKey->pData)) {
      return false;
    } else {
      return strncmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0;
    }
  } else {
    return memcmp(pKey->pData, v, pKey->bytes) == 0;
  }
}

1080
static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
L
Liu Jicong 已提交
1081
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1082
  SExprSupp*     pSup = &pOperator->exprSupp;
1083

1084
  SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
1085 1086 1087
  int64_t          gid = pBlock->info.groupId;

  bool    masterScan = true;
1088
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1089 1090
  int16_t bytes = pStateColInfoData->info.bytes;

1091
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1092 1093 1094 1095 1096
  TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

1097
  struct SColumnDataAgg* pAgg = NULL;
1098
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
X
Xiaoyu Wang 已提交
1099
    pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
1100
    if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
1101 1102 1103 1104 1105 1106
      continue;
    }

    char* val = colDataGetData(pStateColInfoData, j);

    if (!pInfo->hasKey) {
1107 1108 1109 1110 1111 1112 1113
      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }

1114 1115 1116 1117
      pInfo->hasKey = true;

      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
1118
    } else if (compareVal(val, &pInfo->stateKey)) {
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
      doKeepTuple(pRowSup, tsList[j]);
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // a new state window started
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1130 1131
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1132 1133 1134 1135 1136
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
      }

      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1137
      doApplyFunctions(pTaskInfo, pSup->pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1138 1139 1140 1141 1142
                       pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);

      // here we start a new session window
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
1143 1144 1145 1146 1147 1148 1149

      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }
1150 1151 1152 1153 1154
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1155 1156
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1157 1158 1159 1160 1161
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
1162
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1163 1164 1165
                   pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
}

1166
static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
1167 1168 1169 1170 1171
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
1172

1173 1174
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  SExprSupp*     pSup = &pOperator->exprSupp;
1175

1176
  SOptrBasicInfo* pBInfo = &pInfo->binfo;
1177 1178

  if (pOperator->status == OP_RES_TO_RETURN) {
1179
    while (1) {
1180
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1181
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1182

1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193
      bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }

      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1194
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1195 1196
  }

1197
  int32_t order = TSDB_ORDER_ASC;
1198
  int64_t st = taosGetTimestampUs();
1199 1200 1201

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  while (1) {
1202
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1203 1204 1205 1206
    if (pBlock == NULL) {
      break;
    }

1207
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1208 1209
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1210 1211 1212
    doStateWindowAggImpl(pOperator, pInfo, pBlock);
  }

X
Xiaoyu Wang 已提交
1213
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1214

1215 1216 1217
  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pBInfo->resultRowInfo);

1218
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1219
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1220
  while (1) {
1221
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1222
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1223

1224 1225 1226 1227 1228
    bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
1229

1230 1231 1232 1233 1234
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1235
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1236 1237
}

1238
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
1239
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
1240
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1241 1242 1243 1244 1245 1246 1247 1248

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pBlock = pInfo->binfo.pRes;

  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
1249
    return pOperator->fpSet.getStreamResFn(pOperator);
1250 1251 1252 1253 1254 1255 1256
  } else {
    pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
    if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
      return NULL;
    }

    blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
1257 1258
    while (1) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1259
      doFilter(pInfo->pCondition, pBlock, NULL);
1260

1261 1262 1263 1264 1265
      bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
1266

1267 1268 1269
      if (pBlock->info.rows > 0) {
        break;
      }
1270 1271
    }

1272 1273 1274
    size_t rows = pBlock->info.rows;
    pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1275
    return (rows == 0) ? NULL : pBlock;
1276 1277 1278 1279
  }
}

// todo merged with the build group result.
1280
static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
1281
                                  int32_t* rowEntryInfoOffset) {
1282 1283 1284 1285 1286 1287 1288
  size_t num = taosArrayGetSize(pUpdateList);

  for (int32_t i = 0; i < num; ++i) {
    SResKeyPos* pPos = taosArrayGetP(pUpdateList, i);

    SFilePage*  bufPage = getBufPage(pBuf, pPos->pos.pageId);
    SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
1289

1290
    for (int32_t j = 0; j < numOfOutput; ++j) {
1291
      SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, j, rowEntryInfoOffset);
1292 1293
      if (pRow->numOfRows < pEntry->numOfRes) {
        pRow->numOfRows = pEntry->numOfRes;
1294 1295 1296 1297 1298 1299
      }
    }

    releaseBufPage(pBuf, bufPage);
  }
}
5
54liuyao 已提交
1300
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
L
Liu Jicong 已提交
1301
  for (int i = 0; i < num; i++) {
5
54liuyao 已提交
1302 1303
    if (type == STREAM_INVERT) {
      fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
L
Liu Jicong 已提交
1304
    } else if (type == STREAM_NORMAL) {
5
54liuyao 已提交
1305 1306 1307 1308
      fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
    }
  }
}
5
54liuyao 已提交
1309

1310
void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
1311
  SResultRow*     pResult = getResultRowByPos(pResultBuf, p1);
1312
  SqlFunctionCtx* pCtx = pSup->pCtx;
5
54liuyao 已提交
1313
  for (int32_t i = 0; i < numOfOutput; ++i) {
1314
    pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
1315 1316 1317 1318 1319 1320 1321 1322 1323
    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
      continue;
    }
    pResInfo->initialized = false;
    if (pCtx[i].functionId != -1) {
      pCtx[i].fpSet.init(&pCtx[i], pResInfo);
    }
  }
5
54liuyao 已提交
1324 1325 1326
  SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pResultBuf, bufPage);
5
54liuyao 已提交
1327 1328
}

5
54liuyao 已提交
1329
bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
X
Xiaoyu Wang 已提交
1330
                   int32_t numOfOutput) {
1331
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId);
5
54liuyao 已提交
1332
  SResultRowPosition* p1 =
1333
      (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1334 1335
  if (!p1) {
    // window has been closed
5
54liuyao 已提交
1336
    return false;
1337
  }
1338
  doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput);
5
54liuyao 已提交
1339
  return true;
5
54liuyao 已提交
1340 1341
}

1342 1343 1344 1345 1346 1347 1348 1349 1350
bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) {
  size_t bytes = sizeof(TSKEY);
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId);
  SResultRowPosition* p1 =
      (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
  if (!p1) {
    // window has been closed
    return false;
  }
5
54liuyao 已提交
1351
  // SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId);
1352 1353 1354 1355 1356 1357 1358
  // dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage);
  taosHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
  return true;
}

void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pUpWins, SInterval* pInterval) {
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
1359
  TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
1360
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
1361
  uint64_t*        groupIds = (uint64_t*)pGroupCol->pData;
1362 1363 1364
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
1365
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC);
1366 1367 1368 1369 1370 1371 1372 1373
    doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
    if (pUpWins) {
      SWinRes winRes = {.ts = win.skey, .groupId = groupIds[i]};
      taosArrayPush(pUpWins, &winRes);
    }
  }
}

1374 1375
static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput,
                           SSDataBlock* pBlock, SArray* pUpWins) {
1376 1377 1378 1379
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  TSKEY*           startTsCols = (TSKEY*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           endTsCols = (TSKEY*)pEndTsCol->pData;
5
54liuyao 已提交
1380 1381
  uint64_t*        pGpDatas = NULL;
  if (pBlock->info.type == STREAM_RETRIEVE) {
5
54liuyao 已提交
1382
    SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
L
Liu Jicong 已提交
1383
    pGpDatas = (uint64_t*)pGpCol->pData;
5
54liuyao 已提交
1384
  }
1385 1386
  int32_t        step = 0;
  int32_t        startPos = 0;
1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
    while (win.ekey <= endTsCols[i]) {
      uint64_t winGpId = pGpDatas ? pGpDatas[startPos] : pBlock->info.groupId;
      bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput);
      if (pUpWins && res) {
        SWinRes winRes = {.ts = win.skey, .groupId = winGpId};
        taosArrayPush(pUpWins, &winRes);
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
5
54liuyao 已提交
1399
    }
5
54liuyao 已提交
1400 1401
  }
}
1402

5
54liuyao 已提交
1403 1404 1405 1406 1407 1408 1409
static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) {
  void*  pIte = NULL;
  size_t keyLen = 0;
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    void*    key = taosHashGetKey(pIte, &keyLen);
    uint64_t groupId = *(uint64_t*)key;
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
1410
    TSKEY               ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1411
    SResultRowPosition* pPos = (SResultRowPosition*)pIte;
1412
    int32_t             code = saveResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
5
54liuyao 已提交
1413 1414 1415 1416 1417 1418 1419
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }
  return TSDB_CODE_SUCCESS;
}

1420 1421 1422
static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
                                   SHashObj* pPullDataMap, SArray* closeWins, SArray* pRecyPages,
                                   SDiskbasedBuf* pDiscBuf) {
X
Xiaoyu Wang 已提交
1423
  void*  pIte = NULL;
5
54liuyao 已提交
1424
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
1425 1426 1427
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    void*    key = taosHashGetKey(pIte, &keyLen);
    uint64_t groupId = *(uint64_t*)key;
5
54liuyao 已提交
1428
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
L
Liu Jicong 已提交
1429
    TSKEY       ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1430 1431 1432
    STimeWindow win;
    win.skey = ts;
    win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
L
Liu Jicong 已提交
1433 1434 1435
    SWinRes winRe = {
        .ts = win.skey,
        .groupId = groupId,
L
Liu Jicong 已提交
1436
    };
5
54liuyao 已提交
1437
    void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinRes));
5
54liuyao 已提交
1438
    if (isCloseWindow(&win, pSup)) {
5
54liuyao 已提交
1439
      if (chIds && pPullDataMap) {
L
Liu Jicong 已提交
1440
        SArray* chAy = *(SArray**)chIds;
5
54liuyao 已提交
1441
        int32_t size = taosArrayGetSize(chAy);
5
54liuyao 已提交
1442
        qDebug("===stream===window %" PRId64 " wait child size:%d", win.skey, size);
5
54liuyao 已提交
1443
        for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
1444
          qDebug("===stream===window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i));
5
54liuyao 已提交
1445 1446 1447
        }
        continue;
      } else if (pPullDataMap) {
5
54liuyao 已提交
1448
        qDebug("===stream===close window %" PRId64, win.skey);
5
54liuyao 已提交
1449
      }
5
54liuyao 已提交
1450 1451 1452 1453 1454 1455
      SResultRowPosition* pPos = (SResultRowPosition*)pIte;
      if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
        int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins);
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
1456 1457 1458
        ASSERT(pRecyPages != NULL);
        taosArrayPush(pRecyPages, &pPos->pageId);
      } else {
5
54liuyao 已提交
1459
        // SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId);
1460
        // dBufSetBufPageRecycled(pDiscBuf, bufPage);
5
54liuyao 已提交
1461
      }
5
54liuyao 已提交
1462 1463 1464
      char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
      SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
      taosHashRemove(pHashMap, keyBuf, keyLen);
5
54liuyao 已提交
1465 1466 1467 1468 1469
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
1470 1471 1472
static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) {
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
L
Liu Jicong 已提交
1473
    SOperatorInfo*                    pChildOp = taosArrayGetP(pChildren, i);
5
54liuyao 已提交
1474 1475 1476
    SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
    ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
1477 1478
    closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, NULL,
                        pChInfo->aggSup.pResultBuf);
1479 1480 1481 1482 1483 1484 1485
  }
}

static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) {
  int32_t size = taosArrayGetSize(pageIds);
  for (int32_t i = 0; i < size; i++) {
    int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i);
5
54liuyao 已提交
1486
    // SFilePage* bufPage = getBufPage(pDiskBuf, pageId);
1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
    // dBufSetBufPageRecycled(pDiskBuf, bufPage);
  }
  taosArrayClear(pageIds);
}

static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlock) {
  blockDataCleanup(pBlock);
  int32_t size = taosArrayGetSize(pWins);
  if (*index == size) {
    *index = 0;
    taosArrayClear(pWins);
    return;
  }
  blockDataEnsureCapacity(pBlock, size - *index);
  SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
1502
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
1503 1504 1505 1506 1507 1508
  for (int32_t i = *index; i < size; i++) {
    SWinRes* pWin = taosArrayGet(pWins, i);
    colDataAppend(pTsCol, pBlock->info.rows, (const char*)&pWin->ts, false);
    colDataAppend(pGroupCol, pBlock->info.rows, (const char*)&pWin->groupId, false);
    pBlock->info.rows++;
    (*index)++;
5
54liuyao 已提交
1509 1510 1511
  }
}

1512
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
1513
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
1514
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1515 1516

  pInfo->order = TSDB_ORDER_ASC;
1517
  SExprSupp* pSup = &pOperator->exprSupp;
1518 1519 1520 1521 1522 1523

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
1524
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
1525 1526 1527 1528
    if (pInfo->pDelRes->info.rows > 0) {
      return pInfo->pDelRes;
    }

1529
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1530
    if (pInfo->binfo.pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1531
      pOperator->status = OP_EXEC_DONE;
5
54liuyao 已提交
1532
      qDebug("===stream===single interval is done");
1533
      freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
1534
    }
5
54liuyao 已提交
1535
    printDataBlock(pInfo->binfo.pRes, "single interval");
1536 1537 1538 1539 1540
    return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];

1541
  SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);  // SResKeyPos
1542
  while (1) {
1543
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1544 1545 1546
    if (pBlock == NULL) {
      break;
    }
1547
    printDataBlock(pBlock, "single interval recv");
1548

5
54liuyao 已提交
1549
    if (pBlock->info.type == STREAM_CLEAR) {
1550 1551
      doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock,
                     NULL);
1552
      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
5
54liuyao 已提交
1553
      continue;
1554 1555
    }
    if (pBlock->info.type == STREAM_DELETE_DATA) {
1556 1557
      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval);
      continue;
1558
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
1559 1560
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
      continue;
5
54liuyao 已提交
1561
    }
1562

1563 1564 1565 1566 1567
    if (pBlock->info.type == STREAM_NORMAL) {
      //set input version
      pTaskInfo->version = pBlock->info.version;
    }

1568 1569 1570 1571 1572
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }

1573 1574 1575
    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
    // caller. Note that all the time window are not close till now.
    // the pDataBlock are always the same one, no need to call this again
1576
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
1577
    if (pInfo->invertible) {
1578
      setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type);
1579 1580
    }

5
54liuyao 已提交
1581
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
H
Haojun Liao 已提交
1582
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated);
1583
  }
1584
  pOperator->status = OP_RES_TO_RETURN;
1585 1586
  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdated,
                      pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
1587

1588
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
1589 1590
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
1591 1592 1593 1594 1595
  removeDeleteResults(pUpdated, pInfo->pDelWins);
  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows > 0) {
    return pInfo->pDelRes;
  }
1596
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
1597
  printDataBlock(pInfo->binfo.pRes, "single interval");
1598 1599 1600 1601 1602
  return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}

static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
1603
  cleanupBasicInfo(&pInfo->binfo);
1604
  taosMemoryFreeClear(pInfo->stateKey.pData);
1605

D
dapan1121 已提交
1606
  taosMemoryFreeClear(param);
1607 1608
}

H
Haojun Liao 已提交
1609 1610 1611 1612 1613
static void freeItem(void* param) {
  SGroupKeys *pKey = (SGroupKeys*) param;
  taosMemoryFree(pKey->pData);
}

1614
void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) {
1615
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
1616
  cleanupBasicInfo(&pInfo->binfo);
1617
  cleanupAggSup(&pInfo->aggSup);
H
Haojun Liao 已提交
1618 1619 1620 1621 1622 1623 1624
  pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages);
  pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols);
  taosArrayDestroyEx(pInfo->pPrevValues, freeItem);

  pInfo->pPrevValues = NULL;
  pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins);
  pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);
1625

H
Haojun Liao 已提交
1626 1627
  cleanupGroupResInfo(&pInfo->groupResInfo);
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
D
dapan1121 已提交
1628
  taosMemoryFreeClear(param);
1629 1630
}

5
54liuyao 已提交
1631
void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
1632
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
1633
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
1634
  cleanupAggSup(&pInfo->aggSup);
L
Liu Jicong 已提交
1635
  // it should be empty.
5
54liuyao 已提交
1636 1637 1638
  taosHashCleanup(pInfo->pPullDataMap);
  taosArrayDestroy(pInfo->pPullWins);
  blockDataDestroy(pInfo->pPullDataRes);
1639
  taosArrayDestroy(pInfo->pRecycledPages);
H
Haojun Liao 已提交
1640
  blockDataDestroy(pInfo->pUpdateRes);
5
54liuyao 已提交
1641

1642 1643 1644 1645
  if (pInfo->pChildren) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
1646
      destroyStreamFinalIntervalOperatorInfo(pChildOp->info, numOfOutput);
1647 1648 1649
      taosMemoryFreeClear(pChildOp);
    }
  }
1650
  nodesDestroyNode((SNode*)pInfo->pPhyNode);
5
54liuyao 已提交
1651
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
1652

D
dapan1121 已提交
1653
  taosMemoryFreeClear(param);
5
54liuyao 已提交
1654 1655
}

1656
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
5
54liuyao 已提交
1657 1658 1659 1660 1661 1662 1663 1664
  for (int32_t i = 0; i < numOfCols; i++) {
    if (!fmIsInvertible(pFCtx[i].functionId)) {
      return false;
    }
  }
  return true;
}

1665
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
1666 1667
  // the primary timestamp column
  bool needed = false;
1668 1669
  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
1670

X
Xiaoyu Wang 已提交
1671
  {  // ts column
1672 1673
    SColumn c = {0};
    c.colId = 1;
1674
    c.slotId = pInfo->primaryTsIndex;
1675 1676
    c.type = TSDB_DATA_TYPE_TIMESTAMP;
    c.bytes = sizeof(int64_t);
1677
    taosArrayPush(pInfo->pInterpCols, &c);
1678 1679

    SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1680 1681 1682 1683
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = true;  // to denote no value is assigned yet
    key.pData = taosMemoryCalloc(1, c.bytes);
1684
    taosArrayPush(pInfo->pPrevValues, &key);
1685 1686
  }

X
Xiaoyu Wang 已提交
1687
  for (int32_t i = 0; i < numOfCols; ++i) {
1688 1689
    SExprInfo* pExpr = pCtx[i].pExpr;

H
Haojun Liao 已提交
1690
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1691 1692 1693
      SFunctParam* pParam = &pExpr->base.pParam[0];

      SColumn c = *pParam->pCol;
1694
      taosArrayPush(pInfo->pInterpCols, &c);
1695 1696 1697
      needed = true;

      SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1698 1699
      key.bytes = c.bytes;
      key.type = c.type;
1700
      key.isNull = false;
X
Xiaoyu Wang 已提交
1701
      key.pData = taosMemoryCalloc(1, c.bytes);
1702
      taosArrayPush(pInfo->pPrevValues, &key);
1703 1704 1705 1706 1707 1708
    }
  }

  return needed;
}

1709
void increaseTs(SqlFunctionCtx* pCtx) {
1710
  if (pCtx[0].pExpr->pExpr->_function.pFunctNode->funcType == FUNCTION_TYPE_WSTART) {
1711 1712 1713 1714
    pCtx[0].increase = true;
  }
}

1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736
SSDataBlock* createDeleteBlock() {
  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
  pBlock->info.hasVarCol = false;
  pBlock->info.groupId = 0;
  pBlock->info.rows = 0;
  pBlock->info.type = STREAM_DELETE_RESULT;
  pBlock->info.rowSize = sizeof(TSKEY) + sizeof(uint64_t);

  pBlock->pDataBlock = taosArrayInit(2, sizeof(SColumnInfoData));
  SColumnInfoData infoData = {0};
  infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP;
  infoData.info.bytes = sizeof(TSKEY);
  // window start ts
  taosArrayPush(pBlock->pDataBlock, &infoData);

  infoData.info.type = TSDB_DATA_TYPE_UBIGINT;
  infoData.info.bytes = sizeof(uint64_t);
  taosArrayPush(pBlock->pDataBlock, &infoData);

  return pBlock;
}

5
54liuyao 已提交
1737 1738 1739 1740 1741 1742
void initIntervalDownStream(SOperatorInfo* downstream, uint8_t type) {
  ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
  SStreamScanInfo* pScanInfo = downstream->info;
  pScanInfo->sessionSup.parentType = type;
}

1743 1744
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                          SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
L
Liu Jicong 已提交
1745 1746
                                          STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode,
                                          SExecTaskInfo* pTaskInfo, bool isStream) {
1747
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1748
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1749 1750 1751 1752
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

L
Liu Jicong 已提交
1753 1754 1755
  pInfo->win = pTaskInfo->window;
  pInfo->order = TSDB_ORDER_ASC;
  pInfo->interval = *pInterval;
L
Liu Jicong 已提交
1756
  pInfo->execModel = pTaskInfo->execModel;
L
Liu Jicong 已提交
1757
  pInfo->twAggSup = *pTwAggSupp;
5
54liuyao 已提交
1758
  pInfo->ignoreExpiredData = pPhyNode->window.igExpired;
1759
  pInfo->pCondition = pPhyNode->window.node.pConditions;
1760 1761 1762 1763

  if (pPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
L
Liu Jicong 已提交
1764
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
1765 1766 1767 1768
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
1769

1770 1771
  pInfo->primaryTsIndex = primaryTsSlotId;

1772 1773
  SExprSupp* pSup = &pOperator->exprSupp;

1774
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
1775
  initResultSizeInfo(&pOperator->resultInfo, 4096);
1776

1777 1778 1779
  int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);

1780 1781
  if (isStream) {
    ASSERT(numOfCols > 0);
1782
    increaseTs(pSup->pCtx);
1783
  }
1784

1785
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
1786

1787
  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
X
Xiaoyu Wang 已提交
1788
  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
1789

1790
  pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, pInfo);
1791 1792
  if (pInfo->timeWindowInterpo) {
    pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
H
Haojun Liao 已提交
1793 1794 1795
    if (pInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
1796
  }
1797 1798 1799
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes));
  pInfo->delIndex = 0;
1800
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
1801
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1802

X
Xiaoyu Wang 已提交
1803 1804 1805 1806
  pOperator->name = "TimeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
1807
  pOperator->exprSupp.pExprInfo = pExprInfo;
X
Xiaoyu Wang 已提交
1808
  pOperator->pTaskInfo = pTaskInfo;
1809
  pOperator->exprSupp.numOfExprs = numOfCols;
X
Xiaoyu Wang 已提交
1810
  pOperator->info = pInfo;
1811 1812 1813 1814

  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL,
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

5
54liuyao 已提交
1815 1816 1817 1818
  if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) {
    initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL);
  }

1819 1820 1821 1822 1823 1824 1825
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1826
_error:
1827 1828 1829 1830 1831 1832 1833 1834 1835
  destroyIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                                SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
wmmhello's avatar
wmmhello 已提交
1836
                                                STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
1837
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1838
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

  pInfo->order = TSDB_ORDER_ASC;
  pInfo->interval = *pInterval;
  pInfo->execModel = OPTR_EXEC_MODEL_STREAM;
  pInfo->win = pTaskInfo->window;
  pInfo->twAggSup = *pTwAggSupp;
  pInfo->primaryTsIndex = primaryTsSlotId;

  int32_t numOfRows = 4096;
  size_t  keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

1853
  initResultSizeInfo(&pOperator->resultInfo, numOfRows);
1854 1855
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);
1856 1857
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);

wmmhello's avatar
wmmhello 已提交
1858
  if (code != TSDB_CODE_SUCCESS) {
1859 1860 1861
    goto _error;
  }

1862
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1863 1864

  pOperator->name = "StreamTimeIntervalAggOperator";
X
Xiaoyu Wang 已提交
1865
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
1866
  pOperator->blocking = true;
1867
  pOperator->status = OP_NOT_OPENED;
1868
  pOperator->exprSupp.pExprInfo = pExprInfo;
1869
  pOperator->pTaskInfo = pTaskInfo;
1870
  pOperator->exprSupp.numOfExprs = numOfCols;
1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882
  pOperator->info = pInfo;

  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL,
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1883
_error:
1884 1885 1886 1887 1888 1889 1890 1891 1892 1893
  destroyIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

// todo handle multiple tables cases.
static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1894
  SExprSupp*     pSup = &pOperator->exprSupp;
1895

1896
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1897 1898

  bool    masterScan = true;
1899
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930
  int64_t gid = pBlock->info.groupId;

  int64_t gap = pInfo->gap;

  if (!pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->winSup.prevTs = INT64_MIN;
  }

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

  // In case of ascending or descending order scan data, only one time window needs to be kepted for each table.
  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
    if (pInfo->winSup.prevTs == INT64_MIN) {
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
    } else if (tsList[j] - pRowSup->prevTs <= gap && (tsList[j] - pRowSup->prevTs) >= 0) {
      // The gap is less than the threshold, so it belongs to current session window that has been opened already.
      doKeepTuple(pRowSup, tsList[j]);
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1931 1932
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1933 1934 1935 1936 1937 1938
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
      }

      // pInfo->numOfRows data belong to the current session window
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1939
      doApplyFunctions(pTaskInfo, pSup->pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1940 1941 1942 1943 1944 1945 1946 1947 1948 1949
                       pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);

      // here we start a new session window
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1950 1951
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1952 1953 1954 1955 1956
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
1957
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1958 1959 1960
                   pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
}

1961
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
1962 1963 1964 1965 1966 1967
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*          pBInfo = &pInfo->binfo;
1968
  SExprSupp*               pSup = &pOperator->exprSupp;
1969 1970

  if (pOperator->status == OP_RES_TO_RETURN) {
1971
    while (1) {
1972
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1973
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1974 1975 1976 1977 1978 1979

      bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
1980

1981 1982 1983 1984 1985
      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1986
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1987 1988
  }

1989 1990 1991
  int64_t st = taosGetTimestampUs();
  int32_t order = TSDB_ORDER_ASC;

1992 1993 1994
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1995
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1996 1997 1998 1999 2000
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
2001
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
2002 2003
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

2004 2005 2006
    doSessionWindowAggImpl(pOperator, pInfo, pBlock);
  }

2007 2008
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;

2009 2010 2011 2012
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pBInfo->resultRowInfo);

2013
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
2014
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
2015
  while (1) {
2016
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
2017
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
2018

2019 2020 2021 2022 2023
    bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
2024

2025 2026 2027 2028 2029
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
2030
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
2031 2032
}

2033
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
H
Haojun Liao 已提交
2034
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2035 2036
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
H
Haojun Liao 已提交
2037 2038 2039 2040 2041 2042

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);

      pkey->isNull = false;
2043
      char* val = colDataGetData(pColInfoData, rowIndex);
H
Haojun Liao 已提交
2044 2045 2046 2047 2048
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }
}

2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148
static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock,
                                   int32_t rowIndex, SSDataBlock* pResBlock) {
  int32_t rows = pResBlock->info.rows;

  // todo set the correct primary timestamp column

  // output the result
  for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
    SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
    int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
    int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;

    SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
    SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

    switch (pSliceInfo->fillType) {
      case TSDB_FILL_NULL:
        colDataAppendNULL(pDst, rows);
        break;

      case TSDB_FILL_SET_VALUE: {
        SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal;

        if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
          float v = 0;
          GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
          double v = 0;
          GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
          int64_t v = 0;
          GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        }
      } break;

      case TSDB_FILL_LINEAR:
#if 0
        if (pCtx->start.key == INT64_MIN || pCtx->start.key > pCtx->startTs
                    || pCtx->end.key == INT64_MIN || pCtx->end.key < pCtx->startTs) {
//                  goto interp_exit;
                }

              double v1 = -1, v2 = -1;
              GET_TYPED_DATA(v1, double, pCtx->inputType, &pCtx->start.val);
              GET_TYPED_DATA(v2, double, pCtx->inputType, &pCtx->end.val);

              SPoint point1 = {.key = ts, .val = &v1};
              SPoint point2 = {.key = nextTs, .val = &v2};
              SPoint point  = {.key = pCtx->startTs, .val = pCtx->pOutput};

              int32_t srcType = pCtx->inputType;
              if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) {
                setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
              } else {
                bool exceedMax = false, exceedMin = false;
                taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE, &exceedMax, &exceedMin);
                if (exceedMax || exceedMin) {
                  __compar_fn_t func = getComparFunc((int32_t)pCtx->inputType, 0);
                  if (func(&pCtx->start.val, &pCtx->end.val) <= 0) {
                    COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->start.val : &pCtx->end.val);
                  } else {
                    COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->end.val : &pCtx->start.val);
                  }
                }
              }
#endif
        break;

      case TSDB_FILL_PREV: {
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
        colDataAppend(pDst, rows, pkey->pData, false);
      } break;

      case TSDB_FILL_NEXT: {
        char* p = colDataGetData(pSrc, rowIndex);
        colDataAppend(pDst, rows, p, colDataIsNull_s(pSrc, rowIndex));
      } break;

      case TSDB_FILL_NONE:
      default:
        break;
    }
  }

  pResBlock->info.rows += 1;
}

static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pPrevRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pPrevRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

2149
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pPrevRow, &key);
  }

  return TSDB_CODE_SUCCESS;
}

H
Haojun Liao 已提交
2164
static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
2165 2166 2167 2168
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

2169 2170
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

2171
  STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
2172 2173
  SSDataBlock*            pResBlock = pSliceInfo->pRes;
  SExprSupp*              pSup = &pOperator->exprSupp;
H
Haojun Liao 已提交
2174

2175 2176
  blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity);

2177 2178 2179 2180 2181 2182 2183 2184
  //  if (pOperator->status == OP_RES_TO_RETURN) {
  //    //    doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes);
  //    if (pResBlock->info.rows == 0 || !hasDataInGroupInfo(&pSliceInfo->groupResInfo)) {
  //      doSetOperatorCompleted(pOperator);
  //    }
  //
  //    return pResBlock;
  //  }
2185

2186 2187
  int32_t        order = TSDB_ORDER_ASC;
  SInterval*     pInterval = &pSliceInfo->interval;
2188 2189
  SOperatorInfo* downstream = pOperator->pDownstream[0];

H
Haojun Liao 已提交
2190
  int32_t numOfRows = 0;
2191
  while (1) {
2192
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
2193 2194 2195 2196
    if (pBlock == NULL) {
      break;
    }

2197 2198 2199 2200 2201
    int32_t code = initPrevRowsKeeper(pSliceInfo, pBlock);
    if (code != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, code);
    }

2202
    // the pDataBlock are always the same one, no need to call this again
2203
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
H
Haojun Liao 已提交
2204

2205
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
2206 2207
    for (int32_t i = 0; i < pBlock->info.rows; ++i) {
      int64_t ts = *(int64_t*)colDataGetData(pTsCol, i);
H
Haojun Liao 已提交
2208 2209

      if (ts == pSliceInfo->current) {
2210
        for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
2211
          SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
2212 2213
          int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
          int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
H
Haojun Liao 已提交
2214 2215

          SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
2216
          SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
H
Haojun Liao 已提交
2217 2218 2219 2220 2221

          char* v = colDataGetData(pSrc, i);
          colDataAppend(pDst, numOfRows, v, false);
        }

2222
        pResBlock->info.rows += 1;
2223
        doKeepPrevRows(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2224

2225 2226
        pSliceInfo->current =
            taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
H
Haojun Liao 已提交
2227 2228 2229 2230
        if (pSliceInfo->current > pSliceInfo->win.ekey) {
          doSetOperatorCompleted(pOperator);
          break;
        }
2231 2232 2233 2234

        if (pResBlock->info.rows >= pResBlock->info.capacity) {
          break;
        }
H
Haojun Liao 已提交
2235
      } else if (ts < pSliceInfo->current) {
2236
        if (i < pBlock->info.rows - 1) {
2237
          int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
H
Haojun Liao 已提交
2238
          if (nextTs > pSliceInfo->current) {
2239 2240 2241
            while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
              genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock);
              pSliceInfo->current =
H
Haojun Liao 已提交
2242
                  taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2243
              if (pResBlock->info.rows >= pResBlock->info.capacity) {
H
Haojun Liao 已提交
2244 2245
                break;
              }
H
Haojun Liao 已提交
2246
            }
2247 2248 2249 2250

            if (pSliceInfo->current > pSliceInfo->win.ekey) {
              doSetOperatorCompleted(pOperator);
              break;
H
Haojun Liao 已提交
2251 2252
            }
          } else {
H
Haojun Liao 已提交
2253
            // ignore current row, and do nothing
H
Haojun Liao 已提交
2254 2255
          }
        } else {  // it is the last row of current block
2256 2257 2258 2259 2260 2261 2262
          doKeepPrevRows(pSliceInfo, pBlock, i);
        }
      } else {  // ts > pSliceInfo->current
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
          genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock);
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2263 2264 2265
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
2266 2267 2268 2269 2270
        }

        if (pSliceInfo->current > pSliceInfo->win.ekey) {
          doSetOperatorCompleted(pOperator);
          break;
H
Haojun Liao 已提交
2271 2272 2273
        }
      }
    }
2274 2275 2276 2277
  }

  // restore the value
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
2278
  if (pResBlock->info.rows == 0) {
2279 2280 2281
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
2282 2283 2284
  return pResBlock->info.rows == 0 ? NULL : pResBlock;
}

2285
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
2286 2287 2288 2289 2290 2291
  STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pOperator == NULL || pInfo == NULL) {
    goto _error;
  }

2292
  SInterpFuncPhysiNode* pInterpPhyNode = (SInterpFuncPhysiNode*)pPhyNode;
2293
  SExprSupp*            pSup = &pOperator->exprSupp;
2294

2295
  int32_t    numOfExprs = 0;
2296
  SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
2297
  int32_t    code = initExprSupp(pSup, pExprInfo, numOfExprs);
H
Haojun Liao 已提交
2298 2299 2300 2301
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2302
  if (pInterpPhyNode->pExprs != NULL) {
2303
    int32_t    num = 0;
2304 2305 2306 2307 2308 2309 2310 2311 2312
    SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

  pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries);
  pInfo->fillType = convertFillType(pInterpPhyNode->fillMode);
2313
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2314

2315 2316 2317
  pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, (SNodeListNode*)pInterpPhyNode->pFillValues);
  pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  pInfo->win = pInterpPhyNode->timeRange;
2318
  pInfo->interval.interval = pInterpPhyNode->interval;
2319
  pInfo->current = pInfo->win.skey;
H
Haojun Liao 已提交
2320

2321
  pOperator->name = "TimeSliceOperator";
2322
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC;
2323 2324 2325 2326
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
  pOperator->pTaskInfo = pTaskInfo;
2327

2328 2329
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyBasicOperatorInfo, NULL, NULL, NULL);
2330

H
Haojun Liao 已提交
2331
  code = appendDownstream(pOperator, &downstream, 1);
2332 2333
  return pOperator;

L
Liu Jicong 已提交
2334
_error:
2335 2336 2337 2338 2339 2340 2341
  taosMemoryFree(pInfo);
  taosMemoryFree(pOperator);
  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
  return NULL;
}

SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
2342
                                             SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId,
2343
                                             SColumn* pStateKeyCol, SNode* pCondition, SExecTaskInfo* pTaskInfo) {
2344 2345 2346 2347 2348 2349
  SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

2350 2351 2352 2353
  pInfo->stateCol = *pStateKeyCol;
  pInfo->stateKey.type = pInfo->stateCol.type;
  pInfo->stateKey.bytes = pInfo->stateCol.bytes;
  pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
2354
  pInfo->pCondition = pCondition;
2355 2356 2357 2358
  if (pInfo->stateKey.pData == NULL) {
    goto _error;
  }

2359 2360
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

2361
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2362 2363 2364
  initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExpr, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);

2365
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2366

L
Liu Jicong 已提交
2367
  pInfo->twAggSup = *pTwAggSup;
2368 2369
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

X
Xiaoyu Wang 已提交
2370 2371
  pInfo->tsSlotId = tsSlotId;
  pOperator->name = "StateWindowOperator";
2372
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE;
X
Xiaoyu Wang 已提交
2373 2374
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
2375 2376
  pOperator->exprSupp.pExprInfo = pExpr;
  pOperator->exprSupp.numOfExprs = numOfCols;
X
Xiaoyu Wang 已提交
2377 2378
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
2379 2380 2381 2382 2383 2384 2385

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL,
                                         destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  int32_t code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

L
Liu Jicong 已提交
2386
_error:
2387 2388 2389 2390 2391 2392
  pTaskInfo->code = TSDB_CODE_SUCCESS;
  return NULL;
}

void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
2393
  cleanupBasicInfo(&pInfo->binfo);
2394

D
dapan1121 已提交
2395
  taosMemoryFreeClear(param);
2396 2397 2398
}

SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
L
Liu Jicong 已提交
2399
                                            SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId,
2400 2401
                                            STimeWindowAggSupp* pTwAggSupp, SNode* pCondition,
                                            SExecTaskInfo* pTaskInfo) {
2402 2403 2404 2405 2406 2407
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

2408
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
2409
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2410

2411
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
2412 2413 2414 2415
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2416 2417
  initBasicInfo(&pInfo->binfo, pResBlock);

2418
  pInfo->twAggSup = *pTwAggSupp;
2419
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2420 2421
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

L
Liu Jicong 已提交
2422 2423 2424 2425 2426
  pInfo->tsSlotId = tsSlotId;
  pInfo->gap = gap;
  pInfo->binfo.pRes = pResBlock;
  pInfo->winSup.prevTs = INT64_MIN;
  pInfo->reptScan = false;
2427
  pInfo->pCondition = pCondition;
L
Liu Jicong 已提交
2428
  pOperator->name = "SessionWindowAggOperator";
2429
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
2430
  pOperator->blocking = true;
L
Liu Jicong 已提交
2431
  pOperator->status = OP_NOT_OPENED;
2432 2433
  pOperator->exprSupp.pExprInfo = pExprInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
L
Liu Jicong 已提交
2434
  pOperator->info = pInfo;
2435 2436 2437 2438 2439 2440 2441 2442

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL,
                                         destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  pOperator->pTaskInfo = pTaskInfo;

  code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

L
Liu Jicong 已提交
2443
_error:
2444 2445 2446 2447 2448 2449 2450 2451
  if (pInfo != NULL) {
    destroySWindowOperatorInfo(pInfo, numOfCols);
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
2452
}
5
54liuyao 已提交
2453

5
54liuyao 已提交
2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471
void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
                      SExecTaskInfo* pTaskInfo) {
  for (int32_t k = 0; k < numOfOutput; ++k) {
    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
      continue;
    }
    int32_t code = TSDB_CODE_SUCCESS;
    if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
      code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
      if (code != TSDB_CODE_SUCCESS) {
        qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
        pTaskInfo->code = code;
        longjmp(pTaskInfo->env, code);
      }
    }
  }
}

2472 2473 2474 2475 2476 2477 2478 2479
bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) {
  int32_t bytes = sizeof(TSKEY);
  SET_RES_WINDOW_KEY(pSup->keyBuf, &ts, bytes, groupId);
  SResultRowPosition* p1 =
      (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
  return p1 != NULL;
}

2480 2481
static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray,
                                  int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SArray* pUpdated) {
5
54liuyao 已提交
2482
  int32_t size = taosArrayGetSize(pWinArray);
5
54liuyao 已提交
2483 2484 2485
  if (!pInfo->pChildren) {
    return;
  }
5
54liuyao 已提交
2486
  for (int32_t i = 0; i < size; i++) {
2487 2488 2489 2490 2491
    SWinRes*    pWinRes = taosArrayGet(pWinArray, i);
    SResultRow* pCurResult = NULL;
    STimeWindow ParentWin = {.skey = pWinRes->ts, .ekey = pWinRes->ts + 1};
    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &ParentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx,
                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2492
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
2493
    bool    find = true;
5
54liuyao 已提交
2494 2495 2496
    for (int32_t j = 0; j < numOfChildren; j++) {
      SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
2497
      SExprSupp*                pChildSup = &pChildOp->exprSupp;
2498 2499 2500 2501
      if (!hasIntervalWindow(&pChInfo->aggSup, pWinRes->ts, pWinRes->groupId)) {
        continue;
      }
      find = true;
2502
      SResultRow* pChResult = NULL;
2503 2504 2505
      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &ParentWin, true, &pChResult, pWinRes->groupId,
                             pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup,
                             pTaskInfo);
2506
      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo);
5
54liuyao 已提交
2507
    }
2508 2509 2510
    if (find && pUpdated) {
      saveResultRow(pCurResult, pWinRes->groupId, pUpdated);
    }
5
54liuyao 已提交
2511 2512 2513 2514 2515
  }
}

bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
  SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId);
2516 2517
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
                                                            GET_RES_WINDOW_KEY_LEN(sizeof(int64_t)));
5
54liuyao 已提交
2518 2519 2520
  return p1 == NULL;
}

L
Liu Jicong 已提交
2521 2522 2523 2524
int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,
                        STimeWindow* pNextWin) {
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538
  int32_t prevEndPos = forwardRows - 1 + startPos;
  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
}

void addPullWindow(SHashObj* pMap, SWinRes* pWinRes, int32_t size) {
  SArray* childIds = taosArrayInit(8, sizeof(int32_t));
  for (int32_t i = 0; i < size; i++) {
    taosArrayPush(childIds, &i);
  }
  taosHashPut(pMap, pWinRes, sizeof(SWinRes), &childIds, sizeof(void*));
}

static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

5
54liuyao 已提交
2539 2540 2541 2542 2543 2544
STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
  return w;
}

5
54liuyao 已提交
2545
static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
S
shenglian zhou 已提交
2546
                           SArray* pUpdated) {
5
54liuyao 已提交
2547
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
X
Xiaoyu Wang 已提交
2548 2549
  SResultRowInfo*                   pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*                    pTaskInfo = pOperatorInfo->pTaskInfo;
2550 2551
  SExprSupp*                        pSup = &pOperatorInfo->exprSupp;
  int32_t                           numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
2552 2553 2554 2555 2556
  int32_t                           step = 1;
  bool                              ascScan = true;
  TSKEY*                            tsCols = NULL;
  SResultRow*                       pResult = NULL;
  int32_t                           forwardRows = 0;
5
54liuyao 已提交
2557

5
54liuyao 已提交
2558 2559 2560
  ASSERT(pSDataBlock->pDataBlock != NULL);
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
2561

X
Xiaoyu Wang 已提交
2562 2563
  int32_t     startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
5
54liuyao 已提交
2564 2565 2566 2567 2568 2569
  STimeWindow nextWin = {0};
  if (IS_FINAL_OP(pInfo)) {
    nextWin = getFinalTimeWindow(ts, &pInfo->interval);
  } else {
    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order);
  }
5
54liuyao 已提交
2570
  while (1) {
5
54liuyao 已提交
2571
    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
5
54liuyao 已提交
2572
    if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
5
54liuyao 已提交
2573 2574 2575 2576 2577 2578 2579
      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
      if (startPos < 0) {
        break;
      }
      continue;
    }
    if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) {
L
Liu Jicong 已提交
2580 2581 2582 2583 2584
      bool    ignore = true;
      SWinRes winRes = {
          .ts = nextWin.skey,
          .groupId = tableGroupId,
      };
5
54liuyao 已提交
2585 2586 2587 2588 2589
      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
      if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId};
        // add pull data request
        taosArrayPush(pInfo->pPullWins, &pull);
5
54liuyao 已提交
2590 2591 2592
        int32_t size = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, &winRes, size);
        qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size);
5
54liuyao 已提交
2593 2594 2595
      } else {
        int32_t index = -1;
        SArray* chArray = NULL;
5
54liuyao 已提交
2596
        int32_t chId = 0;
5
54liuyao 已提交
2597
        if (chIds) {
L
Liu Jicong 已提交
2598
          chArray = *(void**)chIds;
5
54liuyao 已提交
2599
          chId = getChildIndex(pSDataBlock);
5
54liuyao 已提交
2600 2601
          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
        }
L
Liu Jicong 已提交
2602
        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {
5
54liuyao 已提交
2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613
          ignore = false;
        }
      }

      if (ignore) {
        startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
        if (startPos < 0) {
          break;
        }
        continue;
      }
5
54liuyao 已提交
2614
    }
5
54liuyao 已提交
2615

2616 2617
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx,
                                          numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2618 2619 2620
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
5
54liuyao 已提交
2621

5
54liuyao 已提交
2622 2623 2624
    if (IS_FINAL_OP(pInfo)) {
      forwardRows = 1;
    } else {
H
Hongze Cheng 已提交
2625 2626
      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,
                                             NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2627
    }
5
54liuyao 已提交
2628
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) {
5
54liuyao 已提交
2629
      saveResultRow(pResult, tableGroupId, pUpdated);
5
54liuyao 已提交
2630
    }
5
54liuyao 已提交
2631
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
2632 2633
    doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
                     pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
2634
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
2635
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
5
54liuyao 已提交
2636 2637 2638 2639 2640 2641 2642
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
    if (startPos < 0) {
      break;
    }
  }
}

5
54liuyao 已提交
2643 2644 2645 2646
static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) {
  taosHashClear(pInfo->aggSup.pResultRowHashTable);
  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
  cleanupResultRowInfo(&pInfo->binfo.resultRowInfo);
2647
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
2648 2649
}

5
54liuyao 已提交
2650 2651 2652 2653
static void clearSpecialDataBlock(SSDataBlock* pBlock) {
  if (pBlock->info.rows <= 0) {
    return;
  }
5
54liuyao 已提交
2654 2655 2656
  blockDataCleanup(pBlock);
}

2657
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
5
54liuyao 已提交
2658 2659
  // ASSERT(pDest->info.capacity >= pSource->info.rows);
  blockDataEnsureCapacity(pDest, pSource->info.rows);
5
54liuyao 已提交
2660
  clearSpecialDataBlock(pDest);
5
54liuyao 已提交
2661 2662
  SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
  SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
2663

5
54liuyao 已提交
2664
  // copy timestamp column
2665 2666
  colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info);
  for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) {
5
54liuyao 已提交
2667 2668 2669
    SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
    colDataAppendNNULL(pCol, 0, pSource->info.rows);
  }
2670

5
54liuyao 已提交
2671
  pDest->info.rows = pSource->info.rows;
2672 2673
  pDest->info.groupId = pSource->info.groupId;
  pDest->info.type = pSource->info.type;
5
54liuyao 已提交
2674 2675 2676
  blockDataUpdateTsWindow(pDest, 0);
}

5
54liuyao 已提交
2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690
static bool needBreak(SStreamFinalIntervalOperatorInfo* pInfo) {
  int32_t size = taosArrayGetSize(pInfo->pPullWins);
  if (pInfo->pullIndex < size) {
    return true;
  }
  return false;
}

static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
  clearSpecialDataBlock(pBlock);
  int32_t size = taosArrayGetSize(array);
  if (size - (*pIndex) == 0) {
    return;
  }
L
Liu Jicong 已提交
2691
  blockDataEnsureCapacity(pBlock, size - (*pIndex));
5
54liuyao 已提交
2692
  ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
2693 2694 2695 2696 2697
  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
5
54liuyao 已提交
2698
  for (; (*pIndex) < size; (*pIndex)++) {
L
Liu Jicong 已提交
2699
    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));
5
54liuyao 已提交
2700 2701 2702
    colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
    colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
2703 2704
    colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
5
54liuyao 已提交
2705 2706 2707 2708 2709 2710 2711 2712 2713
    pBlock->info.rows++;
  }
  if ((*pIndex) == size) {
    *pIndex = 0;
    taosArrayClear(array);
  }
  blockDataUpdateTsWindow(pBlock, 0);
}

L
Liu Jicong 已提交
2714
void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
5
54liuyao 已提交
2715
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
L
Liu Jicong 已提交
2716
  TSKEY*           tsData = (TSKEY*)pStartCol->pData;
5
54liuyao 已提交
2717
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
L
Liu Jicong 已提交
2718 2719
  uint64_t*        groupIdData = (uint64_t*)pGroupCol->pData;
  int32_t          chId = getChildIndex(pBlock);
5
54liuyao 已提交
2720 2721
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SWinRes winRes = {.ts = tsData[i], .groupId = groupIdData[i]};
L
Liu Jicong 已提交
2722
    void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes));
5
54liuyao 已提交
2723
    if (chIds) {
L
Liu Jicong 已提交
2724
      SArray* chArray = *(SArray**)chIds;
5
54liuyao 已提交
2725 2726
      int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
      if (index != -1) {
5
54liuyao 已提交
2727
        qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId);
5
54liuyao 已提交
2728 2729 2730 2731 2732 2733 2734 2735 2736
        taosArrayRemove(chArray, index);
        if (taosArrayGetSize(chArray) == 0) {
          // pull data is over
          taosHashRemove(pMap, &winRes, sizeof(SWinRes));
        }
      }
    }
  }
}
5
54liuyao 已提交
2737

5
54liuyao 已提交
2738 2739
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
  SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2740
  SOperatorInfo*                    downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
2741
  SArray*                           pUpdated = taosArrayInit(4, POINTER_BYTES);
5
54liuyao 已提交
2742
  TSKEY                             maxTs = INT64_MIN;
5
54liuyao 已提交
2743

2744 2745
  SExprSupp* pSup = &pOperator->exprSupp;

5
54liuyao 已提交
2746
  qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
L
Liu Jicong 已提交
2747

5
54liuyao 已提交
2748 2749 2750 2751
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
2752
    if (pInfo->binfo.pRes->info.rows == 0) {
5
54liuyao 已提交
2753
      pOperator->status = OP_EXEC_DONE;
5
54liuyao 已提交
2754 2755 2756
      if (!IS_FINAL_OP(pInfo)) {
        // semi interval operator clear disk buffer
        clearStreamIntervalOperator(pInfo);
2757 2758
      } else {
        freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
2759
      }
5
54liuyao 已提交
2760 2761
      return NULL;
    }
5
54liuyao 已提交
2762
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
5
54liuyao 已提交
2763 2764
    return pInfo->binfo.pRes;
  } else {
5
54liuyao 已提交
2765 2766 2767 2768 2769 2770
    if (!IS_FINAL_OP(pInfo)) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
      if (pInfo->binfo.pRes->info.rows != 0) {
        printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
        return pInfo->binfo.pRes;
      }
5
54liuyao 已提交
2771 2772 2773 2774
    }
    if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
      pInfo->returnUpdate = false;
      ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
2775
      printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
5
54liuyao 已提交
2776 2777
      // process the rest of the data
      return pInfo->pUpdateRes;
5
54liuyao 已提交
2778
    }
5
54liuyao 已提交
2779 2780 2781 2782
    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
    if (pInfo->pPullDataRes->info.rows != 0) {
      // process the rest of the data
      ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
2783
      printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
5
54liuyao 已提交
2784 2785
      return pInfo->pPullDataRes;
    }
2786 2787 2788
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
5
54liuyao 已提交
2789
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
2790 2791
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
2792 2793 2794 2795 2796
  }

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
2797
      clearSpecialDataBlock(pInfo->pUpdateRes);
2798
      removeDeleteResults(pUpdated, pInfo->pDelWins);
5
54liuyao 已提交
2799
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
2800
      qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
5
54liuyao 已提交
2801 2802
      break;
    }
5
54liuyao 已提交
2803
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv");
5
54liuyao 已提交
2804
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
2805

L
Liu Jicong 已提交
2806
    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA ||
L
Liu Jicong 已提交
2807
        pBlock->info.type == STREAM_INVALID) {
5
54liuyao 已提交
2808 2809
      pInfo->binfo.pRes->info.type = pBlock->info.type;
    } else if (pBlock->info.type == STREAM_CLEAR) {
2810
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes));
2811
      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
5
54liuyao 已提交
2812
      if (IS_FINAL_OP(pInfo)) {
L
Liu Jicong 已提交
2813 2814
        int32_t                           childIndex = getChildIndex(pBlock);
        SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
2815
        SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
L
Liu Jicong 已提交
2816
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;
2817

2818 2819 2820
        doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildSup->numOfExprs, pBlock, NULL);
        rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs,
                              pOperator->pTaskInfo, NULL);
5
54liuyao 已提交
2821 2822
        taosArrayDestroy(pUpWins);
        continue;
2823
      }
5
54liuyao 已提交
2824
      removeResults(pUpWins, pUpdated);
2825 2826
      copyDataBlock(pInfo->pUpdateRes, pBlock);
      // copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
2827
      pInfo->returnUpdate = true;
2828
      taosArrayDestroy(pUpWins);
5
54liuyao 已提交
2829
      break;
2830 2831 2832 2833 2834 2835 2836 2837 2838
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval);
      if (IS_FINAL_OP(pInfo)) {
        int32_t                           childIndex = getChildIndex(pBlock);
        SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;
        doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval);
        rebuildIntervalWindow(pInfo, pSup, pInfo->pDelWins, pInfo->binfo.pRes->info.groupId,
2839
                              pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated);
2840 2841 2842 2843
        continue;
      }
      removeResults(pInfo->pDelWins, pUpdated);
      break;
5
54liuyao 已提交
2844
    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
2845 2846
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
      continue;
5
54liuyao 已提交
2847
    } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
2848
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes));
5
54liuyao 已提交
2849
      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
5
54liuyao 已提交
2850 2851 2852 2853 2854 2855
      removeResults(pUpWins, pUpdated);
      taosArrayDestroy(pUpWins);
      if (taosArrayGetSize(pUpdated) > 0) {
        break;
      }
      continue;
L
Liu Jicong 已提交
2856 2857
    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) {
      processPullOver(pBlock, pInfo->pPullDataMap);
5
54liuyao 已提交
2858
      continue;
5
54liuyao 已提交
2859
    }
5
54liuyao 已提交
2860

5
54liuyao 已提交
2861 2862 2863 2864
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
2865
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
5
54liuyao 已提交
2866
    doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated);
5
54liuyao 已提交
2867
    if (IS_FINAL_OP(pInfo)) {
S
shenglian zhou 已提交
2868
      int32_t chIndex = getChildIndex(pBlock);
5
54liuyao 已提交
2869 2870 2871 2872 2873 2874 2875
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
        SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
        if (!pChildOp) {
          longjmp(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
        }
2876 2877
        SStreamFinalIntervalOperatorInfo* pTmpInfo = pChildOp->info;
        pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
2878
        taosArrayPush(pInfo->pChildren, &pChildOp);
5
54liuyao 已提交
2879
        qDebug("===stream===add child, id:%d", chIndex);
5
54liuyao 已提交
2880
      }
S
shenglian zhou 已提交
2881
      SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
2882
      SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
2883
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
5
54liuyao 已提交
2884
      doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL);
5
54liuyao 已提交
2885 2886 2887 2888

      if (needBreak(pInfo)) {
        break;
      }
5
54liuyao 已提交
2889 2890
    }
  }
S
shenglian zhou 已提交
2891

5
54liuyao 已提交
2892
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
2893
  if (IS_FINAL_OP(pInfo)) {
2894 2895
    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap,
                        pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
2896
    closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
5
54liuyao 已提交
2897 2898
  }

2899
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
2900 2901 2902
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
2903
  if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
2904
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
5
54liuyao 已提交
2905 2906 2907 2908 2909 2910
    return pInfo->binfo.pRes;
  }

  if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
    pInfo->returnUpdate = false;
    ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
2911
    printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
5
54liuyao 已提交
2912 2913 2914
    // process the rest of the data
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
2915 2916 2917 2918 2919

  doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
  if (pInfo->pPullDataRes->info.rows != 0) {
    // process the rest of the data
    ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
2920
    printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
5
54liuyao 已提交
2921 2922
    return pInfo->pPullDataRes;
  }
2923 2924 2925 2926

  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows != 0) {
    // process the rest of the data
5
54liuyao 已提交
2927
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
2928 2929
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
2930 2931 2932 2933
  // ASSERT(false);
  return NULL;
}

2934
SSDataBlock* createSpecialDataBlock(EStreamType type) {
5
54liuyao 已提交
2935 2936 2937 2938
  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
  pBlock->info.hasVarCol = false;
  pBlock->info.groupId = 0;
  pBlock->info.rows = 0;
2939
  pBlock->info.type = type;
5
54liuyao 已提交
2940 2941
  pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) +
      sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY);
5
54liuyao 已提交
2942

5
54liuyao 已提交
2943
  pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData));
5
54liuyao 已提交
2944 2945 2946 2947 2948 2949 2950 2951 2952 2953
  SColumnInfoData infoData = {0};
  infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP;
  infoData.info.bytes = sizeof(TSKEY);
  // window start ts
  taosArrayPush(pBlock->pDataBlock, &infoData);
  // window end ts
  taosArrayPush(pBlock->pDataBlock, &infoData);

  infoData.info.type = TSDB_DATA_TYPE_UBIGINT;
  infoData.info.bytes = sizeof(uint64_t);
2954 2955 2956 2957 2958 2959 2960 2961
  // uid
  taosArrayPush(pBlock->pDataBlock, &infoData);
  // group id
  taosArrayPush(pBlock->pDataBlock, &infoData);

  // calculate start ts
  taosArrayPush(pBlock->pDataBlock, &infoData);
  // calculate end ts
5
54liuyao 已提交
2962 2963 2964
  taosArrayPush(pBlock->pDataBlock, &infoData);

  return pBlock;
5
54liuyao 已提交
2965 2966
}

S
shenglian zhou 已提交
2967 2968 2969
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                     SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
  SIntervalPhysiNode*               pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
5
54liuyao 已提交
2970
  SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo));
S
shenglian zhou 已提交
2971
  SOperatorInfo*                    pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
2972 2973 2974
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
2975

5
54liuyao 已提交
2976
  pInfo->order = TSDB_ORDER_ASC;
S
shenglian zhou 已提交
2977 2978 2979 2980 2981 2982 2983 2984
  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                .sliding = pIntervalPhyNode->sliding,
                                .intervalUnit = pIntervalPhyNode->intervalUnit,
                                .slidingUnit = pIntervalPhyNode->slidingUnit,
                                .offset = pIntervalPhyNode->offset,
                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pIntervalPhyNode->window.watermark,
5
54liuyao 已提交
2985 2986
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
S
shenglian zhou 已提交
2987
  };
2988
  ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY);
5
54liuyao 已提交
2989 2990
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
2991
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
2992 2993 2994 2995 2996 2997 2998 2999 3000
  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

S
shenglian zhou 已提交
3001 3002
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5
54liuyao 已提交
3003
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
3004 3005 3006 3007

  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);

3008
  ASSERT(numOfCols > 0);
3009
  increaseTs(pOperator->exprSupp.pCtx);
5
54liuyao 已提交
3010 3011 3012 3013
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
3014
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3015 3016
  pInfo->pChildren = NULL;
  if (numOfChild > 0) {
3017
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
5
54liuyao 已提交
3018 3019 3020
    for (int32_t i = 0; i < numOfChild; i++) {
      SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp) {
3021 3022
        SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
        pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3023 3024 3025 3026 3027 3028
        taosArrayPush(pInfo->pChildren, &pChildOp);
        continue;
      }
      goto _error;
    }
  }
3029
  pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
3030
  blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
5
54liuyao 已提交
3031 3032
  pInfo->returnUpdate = false;

3033
  pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
5
54liuyao 已提交
3034

5
54liuyao 已提交
3035 3036 3037 3038
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
    pInfo->isFinal = true;
    pOperator->name = "StreamFinalIntervalOperator";
  } else {
5
54liuyao 已提交
3039
    // semi interval operator does not catch result
5
54liuyao 已提交
3040 3041 3042 3043
    pInfo->isFinal = false;
    pOperator->name = "StreamSemiIntervalOperator";
  }

5
54liuyao 已提交
3044
  if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
5
54liuyao 已提交
3045 3046
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }
5
54liuyao 已提交
3047 3048 3049 3050
  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
  pInfo->pullIndex = 0;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
3051
  pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
5
54liuyao 已提交
3052
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
3053
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3054 3055
  pInfo->delIndex = 0;
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes));
5
54liuyao 已提交
3056

5
54liuyao 已提交
3057
  pOperator->operatorType = pPhyNode->type;
5
54liuyao 已提交
3058 3059
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
3060
  pOperator->exprSupp.pExprInfo = pExprInfo;
5
54liuyao 已提交
3061
  pOperator->pTaskInfo = pTaskInfo;
3062
  pOperator->exprSupp.numOfExprs = numOfCols;
5
54liuyao 已提交
3063 3064
  pOperator->info = pInfo;

S
shenglian zhou 已提交
3065 3066 3067
  pOperator->fpSet =
      createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
3068 3069 3070
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
    initIntervalDownStream(downstream, pPhyNode->type);
  }
5
54liuyao 已提交
3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
  destroyStreamFinalIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
5
54liuyao 已提交
3084
}
5
54liuyao 已提交
3085 3086 3087

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
3088
  void** pIte = NULL;
3089
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
3090
    SArray* pWins = (SArray*)(*pIte);
3091 3092 3093
    taosArrayDestroy(pWins);
  }
  taosHashCleanup(pSup->pResultRows);
5
54liuyao 已提交
3094
  destroyDiskbasedBuf(pSup->pResultBuf);
3095
  blockDataDestroy(pSup->pScanBlock);
5
54liuyao 已提交
3096 3097 3098 3099
}

void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;
3100
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
3101 3102
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
3103 3104 3105
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
3106
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
3107 3108 3109 3110 3111 3112
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
      destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput);
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
5
54liuyao 已提交
3113 3114 3115 3116
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  blockDataDestroy(pInfo->pWinBlock);
  blockDataDestroy(pInfo->pUpdateRes);
3117

D
dapan1121 已提交
3118
  taosMemoryFreeClear(param);
5
54liuyao 已提交
3119 3120
}

3121 3122
int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,
                        SSDataBlock* pResultBlock) {
3123 3124 3125 3126 3127
  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
  if (code != TSDB_CODE_SUCCESS) {
    return code;
  }

3128
  initBasicInfo(pBasicInfo, pResultBlock);
3129

5
54liuyao 已提交
3130
  for (int32_t i = 0; i < numOfCols; ++i) {
3131
    pSup->pCtx[i].pBuf = NULL;
5
54liuyao 已提交
3132
  }
3133

3134
  ASSERT(numOfCols > 0);
3135
  increaseTs(pSup->pCtx);
5
54liuyao 已提交
3136 3137 3138 3139 3140 3141 3142 3143
  return TSDB_CODE_SUCCESS;
}

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {
  for (int i = 0; i < nums; i++) {
    pDummy[i].functionId = pCtx[i].functionId;
  }
}
5
54liuyao 已提交
3144

X
Xiaoyu Wang 已提交
3145 3146
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
                    uint8_t type) {
5
54liuyao 已提交
3147
  ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
3148
  SStreamScanInfo* pScanInfo = downstream->info;
X
Xiaoyu Wang 已提交
3149
  pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
5
54liuyao 已提交
3150
  pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
5
54liuyao 已提交
3151 3152
}

3153 3154
int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx,
                                int32_t numOfOutput) {
3155 3156 3157
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo));
}

3158 3159 3160 3161 3162 3163 3164
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                  SExecTaskInfo* pTaskInfo) {
  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
  int32_t                        numOfCols = 0;
  SExprInfo*                     pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock*                   pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;
X
Xiaoyu Wang 已提交
3165
  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
3166
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3167 3168 3169 3170
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

3171
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3172 3173 3174 3175 3176 3177 3178 3179
  if (pSessionNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
3180
  SExprSupp* pSup = &pOperator->exprSupp;
5
54liuyao 已提交
3181

3182
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
3183 3184 3185
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
3186

3187
  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
3188 3189 3190
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
X
Xiaoyu Wang 已提交
3191

5
54liuyao 已提交
3192 3193 3194 3195
  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }
3196
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
3197

3198 3199
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType, .maxTs = INT64_MIN};
H
Haojun Liao 已提交
3200 3201

  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3202 3203
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

5
54liuyao 已提交
3204 3205 3206 3207
  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  if (pSessionNode->window.pTsEnd) {
    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;
  }
3208
  pInfo->gap = pSessionNode->gap;
5
54liuyao 已提交
3209 3210 3211 3212 3213
  pInfo->binfo.pRes = pResBlock;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
3214
  // pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3215 3216
  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete
3217
  pInfo->pChildren = NULL;
5
54liuyao 已提交
3218 3219
  pInfo->isFinal = false;
  pInfo->pPhyNode = pPhyNode;
5
54liuyao 已提交
3220
  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
5
54liuyao 已提交
3221
  pInfo->returnDelete = false;
5
54liuyao 已提交
3222 3223

  pOperator->name = "StreamSessionWindowAggOperator";
3224
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
5
54liuyao 已提交
3225 3226
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
3227 3228
  pOperator->exprSupp.pExprInfo = pExprInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
5
54liuyao 已提交
3229
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
3230 3231 3232
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
3233
  pOperator->pTaskInfo = pTaskInfo;
5
54liuyao 已提交
3234 3235 3236 3237
  if (downstream) {
    initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType);
    code = appendDownstream(pOperator, &downstream, 1);
  }
5
54liuyao 已提交
3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251
  return pOperator;

_error:
  if (pInfo != NULL) {
    destroyStreamSessionAggOperatorInfo(pInfo, numOfCols);
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

int64_t getSessionWindowEndkey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
3252
  SArray*            pWinInfos = (SArray*)data;
5
54liuyao 已提交
3253 3254 3255
  SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index);
  return pWin->win.ekey;
}
5
54liuyao 已提交
3256 3257

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {
5
54liuyao 已提交
3258
  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {
5
54liuyao 已提交
3259 3260 3261 3262 3263
    return true;
  }
  return false;
}

3264
bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); }
5
54liuyao 已提交
3265

X
Xiaoyu Wang 已提交
3266 3267
static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) {
  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
5
54liuyao 已提交
3268 3269 3270 3271
  return taosArrayInsert(pWinInfos, index, &win);
}

static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) {
X
Xiaoyu Wang 已提交
3272
  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
5
54liuyao 已提交
3273 3274 3275
  return taosArrayPush(pWinInfos, &win);
}

3276
SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) {
3277
  void**  ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t));
3278 3279 3280
  SArray* pWinInfos = NULL;
  if (ite == NULL) {
    pWinInfos = taosArrayInit(1024, pAggSup->valueSize);
3281
    taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void*));
3282 3283 3284 3285 3286 3287
  } else {
    pWinInfos = *ite;
  }
  return pWinInfos;
}

5
54liuyao 已提交
3288 3289
// don't add new window
SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
3290
                                       int64_t gap, int32_t* pIndex) {
5
54liuyao 已提交
3291 3292 3293 3294 3295 3296 3297 3298
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    return NULL;
  }
  // find the first position which is smaller than the key
3299
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isInWindow(pWin, startTs, gap)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isInWindow(pWin, startTs, gap)) {
      *pIndex = index + 1;
      return pWin;
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
3315
      *pIndex = index + 1;
5
54liuyao 已提交
3316 3317 3318 3319 3320 3321 3322
      return pWin;
    }
  }

  return NULL;
}

3323 3324
SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
                                        int64_t gap, int32_t* pIndex) {
3325 3326 3327
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

5
54liuyao 已提交
3328 3329
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
5
54liuyao 已提交
3330
    *pIndex = 0;
5
54liuyao 已提交
3331
    return addNewSessionWindow(pWinInfos, startTs);
5
54liuyao 已提交
3332 3333
  }
  // find the first position which is smaller than the key
5
54liuyao 已提交
3334
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3335 3336 3337
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
5
54liuyao 已提交
3338
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3339 3340 3341 3342 3343 3344 3345
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
5
54liuyao 已提交
3346
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3347 3348
      *pIndex = index + 1;
      return pWin;
5
54liuyao 已提交
3349 3350 3351
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
      *pIndex = index;
      return pWin;
5
54liuyao 已提交
3352 3353 3354 3355 3356
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
5
54liuyao 已提交
3357
    return addNewSessionWindow(pWinInfos, startTs);
5
54liuyao 已提交
3358
  }
5
54liuyao 已提交
3359
  *pIndex = index + 1;
5
54liuyao 已提交
3360
  return insertNewSessionWindow(pWinInfos, startTs, index + 1);
5
54liuyao 已提交
3361 3362
}

3363 3364
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows,
                                int32_t start, int64_t gap, SHashObj* pStDeleted) {
5
54liuyao 已提交
3365
  for (int32_t i = start; i < rows; ++i) {
3366
    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {
5
54liuyao 已提交
3367 3368
      return i - start;
    }
5
54liuyao 已提交
3369
    if (pWinInfo->win.skey > pStartTs[i]) {
5
54liuyao 已提交
3370 3371 3372 3373
      if (pStDeleted && pWinInfo->isOutput) {
        taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY));
        pWinInfo->isOutput = false;
      }
5
54liuyao 已提交
3374 3375 3376 3377 3378
      pWinInfo->win.skey = pStartTs[i];
    }
    pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pStartTs[i]);
    if (pEndTs) {
      pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pEndTs[i]);
5
54liuyao 已提交
3379 3380 3381 3382 3383
    }
  }
  return rows - start;
}

X
Xiaoyu Wang 已提交
3384
static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,
3385
                                  uint64_t groupId, int32_t numOfOutput, int32_t* rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3386
                                  SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) {
5
54liuyao 已提交
3387 3388
  assert(pWinInfo->win.skey <= pWinInfo->win.ekey);
  // too many time window in query
3389
  int32_t size = taosArrayGetSize(pAggSup->pCurWins);
5
54liuyao 已提交
3390 3391 3392
  if (size > MAX_INTERVAL_TIME_WINDOW) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
  }
X
Xiaoyu Wang 已提交
3393

5
54liuyao 已提交
3394
  if (pWinInfo->pos.pageId == -1) {
3395
    *pResult = getNewResultRow(pAggSup->pResultBuf, groupId, pAggSup->resultRowSize);
5
54liuyao 已提交
3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413
    if (*pResult == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }
    initResultRow(*pResult);

    // add a new result set for a new group
    pWinInfo->pos.pageId = (*pResult)->pageId;
    pWinInfo->pos.offset = (*pResult)->offset;
  } else {
    *pResult = getResultRowByPos(pAggSup->pResultBuf, &pWinInfo->pos);
    if (!(*pResult)) {
      qError("getResultRowByPos return NULL, TID:%s", GET_TASKID(pTaskInfo));
      return TSDB_CODE_FAILED;
    }
  }

  // set time window for current result
  (*pResult)->win = pWinInfo->win;
3414
  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
5
54liuyao 已提交
3415 3416 3417
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3418 3419 3420
static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup,
                                  SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock,
                                  SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3421
                                  int32_t numOutput, SOperatorInfo* pOperator) {
3422
  SExprSupp*     pSup = &pOperator->exprSupp;
3423 3424
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

X
Xiaoyu Wang 已提交
3425 3426
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId);
  TSKEY*           tsCols = (int64_t*)pColDataInfo->pData;
3427 3428
  int32_t          code = setWindowOutputBuf(pCurWin, pResult, pSup->pCtx, pSDataBlock->info.groupId, numOutput,
                                             pSup->rowEntryInfoOffset, pAggSup, pTaskInfo);
5
54liuyao 已提交
3429 3430 3431
  if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }
5
54liuyao 已提交
3432
  updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false);
3433
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols,
X
Xiaoyu Wang 已提交
3434
                   pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC);
5
54liuyao 已提交
3435 3436 3437
  SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pCurWin->pos.pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pAggSup->pResultBuf, bufPage);
5
54liuyao 已提交
3438 3439 3440
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3441 3442
static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                              SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3443
                              int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3444
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3445
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3446 3447
}

X
Xiaoyu Wang 已提交
3448 3449
static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                                   SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex,
3450
                                   int32_t winRows, int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3451
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3452
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3453 3454
}

5
54liuyao 已提交
3455 3456
int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) {
  SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex);
X
Xiaoyu Wang 已提交
3457
  int32_t            size = taosArrayGetSize(pWinInfos);
5
54liuyao 已提交
3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i);
    if (!isInWindow(pCurWin, pWinInfo->win.skey, gap)) {
      return i - startIndex - 1;
    }
  }

  return size - startIndex - 1;
}

5
54liuyao 已提交
3469
void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId,
3470
                       int32_t numOfOutput, SHashObj* pStUpdated, SHashObj* pStDeleted, SOperatorInfo* pOperator) {
3471
  SExprSupp*     pSup = &pOperator->exprSupp;
3472 3473
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

3474
  SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex);
X
Xiaoyu Wang 已提交
3475
  SResultRow*        pCurResult = NULL;
3476
  setWindowOutputBuf(pCurWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3477
                     &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3478
  num += startIndex + 1;
3479
  ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins));
5
54liuyao 已提交
3480 3481
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < num; i++) {
3482
    SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i);
X
Xiaoyu Wang 已提交
3483
    SResultRow*        pWinResult = NULL;
3484
    setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3485
                       &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3486
    pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey);
3487
    compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo);
5
54liuyao 已提交
3488 3489 3490 3491 3492
    taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition));
    if (pWinInfo->isOutput) {
      taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY));
      pWinInfo->isOutput = false;
    }
3493
    taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
5
54liuyao 已提交
3494 3495
    SFilePage* tmpPage = getBufPage(pInfo->streamAggSup.pResultBuf, pWinInfo->pos.pageId);
    releaseBufPage(pInfo->streamAggSup.pResultBuf, tmpPage);
5
54liuyao 已提交
3496
  }
5
54liuyao 已提交
3497 3498 3499 3500
  SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pCurWin->pos.pageId);
  ASSERT(num > 0);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
5
54liuyao 已提交
3501 3502
}

3503 3504
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
                                   SHashObj* pStDeleted, bool hasEndTs) {
X
Xiaoyu Wang 已提交
3505
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3506
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3507
  bool                           masterScan = true;
3508
  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
5
54liuyao 已提交
3509
  uint64_t                       groupId = pSDataBlock->info.groupId;
X
Xiaoyu Wang 已提交
3510 3511 3512 3513 3514
  int64_t                        gap = pInfo->gap;
  int64_t                        code = TSDB_CODE_SUCCESS;

  int32_t     step = 1;
  bool        ascScan = true;
5
54liuyao 已提交
3515 3516
  TSKEY*      startTsCols = NULL;
  TSKEY*      endTsCols = NULL;
5
54liuyao 已提交
3517
  SResultRow* pResult = NULL;
X
Xiaoyu Wang 已提交
3518
  int32_t     winRows = 0;
5
54liuyao 已提交
3519

5
54liuyao 已提交
3520 3521 3522 3523 3524 3525
  ASSERT(pSDataBlock->pDataBlock);
  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  startTsCols = (int64_t*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = NULL;
  if (hasEndTs) {
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);
5
54liuyao 已提交
3526
  } else {
5
54liuyao 已提交
3527
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
3528
  }
5
54liuyao 已提交
3529
  endTsCols = (int64_t*)pEndTsCol->pData;
X
Xiaoyu Wang 已提交
3530

5
54liuyao 已提交
3531
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
3532
  for (int32_t i = 0; i < pSDataBlock->info.rows;) {
5
54liuyao 已提交
3533
    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
3534 3535 3536
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
3537
    int32_t            winIndex = 0;
3538 3539 3540
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex);
    winRows =
        updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted);
3541
    code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
3542 3543 3544
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
5
54liuyao 已提交
3545

3546
    int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap);
5
54liuyao 已提交
3547
    if (winNum > 0) {
3548
      compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, pStDeleted, pOperator);
5
54liuyao 已提交
3549
    }
5
54liuyao 已提交
3550
    pCurWin->isClosed = false;
5
54liuyao 已提交
3551
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
5
54liuyao 已提交
3552 3553
      SWinRes value = {.ts = pCurWin->win.skey, .groupId = groupId};
      code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinRes));
5
54liuyao 已提交
3554 3555 3556 3557
      if (code != TSDB_CODE_SUCCESS) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
      pCurWin->isOutput = true;
5
54liuyao 已提交
3558 3559 3560 3561 3562
    }
    i += winRows;
  }
}

5
54liuyao 已提交
3563 3564 3565 3566 3567
void deleteWindow(SArray* pWinInfos, int32_t index) {
  ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos));
  taosArrayRemove(pWinInfos, index);
}

3568
static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) {
5
54liuyao 已提交
3569
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
3570
  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;
5
54liuyao 已提交
3571
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
3572
  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;
3573
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
3574
  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;
5
54liuyao 已提交
3575 3576
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    int32_t winIndex = 0;
3577 3578
    while (1) {
      SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex);
5
54liuyao 已提交
3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589
      if (!pCurWin) {
        break;
      }
      deleteWindow(pAggSup->pCurWins, winIndex);
      if (result) {
        taosArrayPush(result, pCurWin);
      }
    }
  }
}

3590 3591
static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock, int32_t tsIndex,
                                  int32_t numOfOutput, int64_t gap, SArray* result) {
5
54liuyao 已提交
3592
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
3593 3594
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
  int32_t          step = 0;
5
54liuyao 已提交
3595
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
3596
    int32_t            winIndex = 0;
3597
    SResultWindowInfo* pCurWin =
5
54liuyao 已提交
3598 3599
        getCurSessionWindow(pAggSup, tsCols[i], INT64_MIN, pBlock->info.groupId, gap, &winIndex);
    if (!pCurWin || pCurWin->pos.pageId == -1) {
5
54liuyao 已提交
3600
      // window has been closed.
5
54liuyao 已提交
3601
      step = 1;
5
54liuyao 已提交
3602 3603
      continue;
    }
5
54liuyao 已提交
3604 3605
    step = updateSessionWindowInfo(pCurWin, tsCols, NULL, pBlock->info.rows, i, gap, NULL);
    ASSERT(isInWindow(pCurWin, tsCols[i], gap));
3606
    doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
3607 3608 3609
    if (result) {
      taosArrayPush(result, pCurWin);
    }
5
54liuyao 已提交
3610 3611 3612
  }
}

5
54liuyao 已提交
3613
static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) {
X
Xiaoyu Wang 已提交
3614
  void*  pData = NULL;
5
54liuyao 已提交
3615
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
3616
  while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) {
5
54liuyao 已提交
3617 3618 3619 3620 3621 3622
    void* key = taosHashGetKey(pData, &keyLen);
    ASSERT(keyLen == sizeof(SResultRowPosition));
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    if (pos == NULL) {
      return TSDB_CODE_QRY_OUT_OF_MEMORY;
    }
5
54liuyao 已提交
3623
    pos->groupId = ((SWinRes*)pData)->groupId;
5
54liuyao 已提交
3624
    pos->pos = *(SResultRowPosition*)key;
5
54liuyao 已提交
3625
    *(int64_t*)pos->key = ((SWinRes*)pData)->ts;
5
54liuyao 已提交
3626 3627
    taosArrayPush(pUpdated, &pos);
  }
5
54liuyao 已提交
3628
  taosArraySort(pUpdated, resultrowComparAsc);
5
54liuyao 已提交
3629 3630 3631 3632 3633
  return TSDB_CODE_SUCCESS;
}

void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
  blockDataCleanup(pBlock);
3634 3635 3636 3637 3638
  int32_t size = taosHashGetSize(pStDeleted);
  if (size == 0) {
    return;
  }
  blockDataEnsureCapacity(pBlock, size);
5
54liuyao 已提交
3639
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
3640
  while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
5
54liuyao 已提交
3641
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
X
Xiaoyu Wang 已提交
3642
    colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false);
3643
    for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); i++) {
5
54liuyao 已提交
3644
      pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
X
Xiaoyu Wang 已提交
3645
      colDataAppendNULL(pColInfoData, pBlock->info.rows);
5
54liuyao 已提交
3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656
    }
    pBlock->info.rows += 1;
    if (pBlock->info.rows + 1 >= pBlock->info.capacity) {
      break;
    }
  }
  if ((*Ite) == NULL) {
    taosHashClear(pStDeleted);
  }
}

X
Xiaoyu Wang 已提交
3657
static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId,
3658
                              int32_t numOfOutput, SOperatorInfo* pOperator) {
3659
  SExprSupp*     pSup = &pOperator->exprSupp;
3660 3661
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

3662 3663
  int32_t size = taosArrayGetSize(pWinArray);
  ASSERT(pInfo->pChildren);
3664

3665 3666
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i);
X
Xiaoyu Wang 已提交
3667
    SResultRow*        pCurResult = NULL;
3668
    setWindowOutputBuf(pParentWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3669
                       &pInfo->streamAggSup, pTaskInfo);
3670 3671
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
    for (int32_t j = 0; j < numOfChildren; j++) {
X
Xiaoyu Wang 已提交
3672
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
3673
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
3674
      SArray*                        pChWins = getWinInfos(&pChInfo->streamAggSup, groupId);
X
Xiaoyu Wang 已提交
3675 3676
      int32_t                        chWinSize = taosArrayGetSize(pChWins);
      int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3677
      if (index < 0) {
3678
        index = 0;
5
54liuyao 已提交
3679 3680
      }
      for (int32_t k = index; k < chWinSize; k++) {
5
54liuyao 已提交
3681 3682
        SResultWindowInfo* pChWin = taosArrayGet(pChWins, k);
        if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) {
3683
          SResultRow* pChResult = NULL;
5
54liuyao 已提交
3684
          setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
3685 3686
                             pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
          compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo);
5
54liuyao 已提交
3687 3688
          SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
          releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage);
3689
          continue;
5
54liuyao 已提交
3690 3691
        } else if (!pChWin->isClosed) {
          break;
3692 3693 3694
        }
      }
    }
5
54liuyao 已提交
3695 3696 3697 3698
    SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId);
    ASSERT(size > 0);
    setBufPageDirty(bufPage, true);
    releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
3699 3700 3701
  }
}

X
Xiaoyu Wang 已提交
3702
typedef SResultWindowInfo* (*__get_win_info_)(void*);
5
54liuyao 已提交
3703 3704
SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
5
54liuyao 已提交
3705

3706 3707
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn,
                           bool delete) {
5
54liuyao 已提交
3708
  // Todo(liuyao) save window to tdb
3709
  void** pIte = NULL;
5
54liuyao 已提交
3710
  size_t keyLen = 0;
3711
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
5
54liuyao 已提交
3712
    uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen);
3713 3714
    SArray*   pWins = (SArray*)(*pIte);
    int32_t   size = taosArrayGetSize(pWins);
3715 3716 3717
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
5
54liuyao 已提交
3718
      if (isCloseWindow(&pSeWin->win, pTwSup)) {
3719 3720
        if (!pSeWin->isClosed) {
          pSeWin->isClosed = true;
5
54liuyao 已提交
3721
          if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {
5
54liuyao 已提交
3722
            int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed);
5
54liuyao 已提交
3723 3724 3725
            if (code != TSDB_CODE_SUCCESS) {
              return code;
            }
3726 3727
            pSeWin->isOutput = true;
          }
5
54liuyao 已提交
3728
          if (delete) {
5
54liuyao 已提交
3729
            deleteWindow(pWins, i);
5
54liuyao 已提交
3730 3731 3732
            i--;
            size = taosArrayGetSize(pWins);
          }
5
54liuyao 已提交
3733
        }
3734
        continue;
5
54liuyao 已提交
3735
      }
3736
      break;
5
54liuyao 已提交
3737 3738 3739 3740 3741
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
3742 3743 3744 3745 3746 3747 3748 3749 3750 3751
static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs, bool delete) {
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);
    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
    closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL, getResWinForSession, delete);
  }
}

3752
int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) {
3753
  void** pIte = NULL;
3754
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
3755
    SArray* pWins = (SArray*)(*pIte);
3756 3757 3758 3759 3760 3761 3762 3763
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (!pSeWin->isClosed) {
        int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed);
        pSeWin->isOutput = true;
      }
5
54liuyao 已提交
3764 3765 3766 3767 3768
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
3769 3770 3771 3772 3773 3774 3775 3776
static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) {
  int32_t size = taosArrayGetSize(pResWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i);
    taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY));
  }
}

5
54liuyao 已提交
3777
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
5
54liuyao 已提交
3778
  SExprSupp*                     pSup = &pOperator->exprSupp;
5
54liuyao 已提交
3779
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3780
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
5
54liuyao 已提交
3781 3782 3783 3784
  TSKEY                          maxTs = INT64_MIN;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
3785 3786
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
L
Liu Jicong 已提交
3787
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "Final  Session" : "Single Session");
5
54liuyao 已提交
3788 3789
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
3790
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
3791
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
5
54liuyao 已提交
3792 3793
      doSetOperatorCompleted(pOperator);
    }
L
Liu Jicong 已提交
3794
    printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "Final  Session" : "Single Session");
5
54liuyao 已提交
3795 3796 3797
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
3798 3799
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
3800
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
3801
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
3802 3803 3804 3805 3806
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
3807
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "Final  Session Recv" : "Single Session Recv");
3808

5
54liuyao 已提交
3809
    if (pBlock->info.type == STREAM_CLEAR) {
X
Xiaoyu Wang 已提交
3810
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
3811 3812
      doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, 0,
                            pWins);
5
54liuyao 已提交
3813 3814
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
X
Xiaoyu Wang 已提交
3815
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
3816
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
3817
        doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, 0, pChildOp->exprSupp.numOfExprs,
5
54liuyao 已提交
3818
                              0, NULL);
3819
        rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
3820 3821
      }
      taosArrayDestroy(pWins);
5
54liuyao 已提交
3822
      continue;
5
54liuyao 已提交
3823 3824 3825
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      // gap must be 0
3826
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins);
5
54liuyao 已提交
3827 3828 3829 3830 3831
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
        // gap must be 0
3832
        doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL);
5
54liuyao 已提交
3833 3834 3835 3836 3837
        rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
      }
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
      taosArrayDestroy(pWins);
      continue;
3838
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
3839
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
5
54liuyao 已提交
3840
      continue;
5
54liuyao 已提交
3841
    }
5
54liuyao 已提交
3842

5
54liuyao 已提交
3843 3844 3845 3846
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
3847
    // the pDataBlock are always the same one, no need to call this again
3848
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
3849 3850 3851 3852 3853 3854
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo));
    if (IS_FINAL_OP(pInfo)) {
      int32_t chIndex = getChildIndex(pBlock);
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
3855 3856
        SOperatorInfo* pChildOp =
            createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
5
54liuyao 已提交
3857 3858 3859 3860 3861
        if (!pChildOp) {
          longjmp(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
        }
        taosArrayPush(pInfo->pChildren, &pChildOp);
      }
3862
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
3863 3864
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true);
3865
    }
5
54liuyao 已提交
3866
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
3867
  }
5
54liuyao 已提交
3868 3869

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
3870 3871
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
3872

3873 3874
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession,
                     pInfo->ignoreExpiredData);
5
54liuyao 已提交
3875
  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
5
54liuyao 已提交
3876 3877 3878
  copyUpdateResult(pStUpdated, pUpdated);
  taosHashCleanup(pStUpdated);

3879
  finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
3880 3881 3882 3883
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
L
Liu Jicong 已提交
3884
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "Final  Session" : "Single Session");
5
54liuyao 已提交
3885 3886 3887
    return pInfo->pDelRes;
  }
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
L
Liu Jicong 已提交
3888
  printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "Final  Session" : "Single Session");
5
54liuyao 已提交
3889 3890 3891 3892
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
3893
  void** pIte = NULL;
5
54liuyao 已提交
3894
  while ((pIte = taosHashIterate(pInfo->streamAggSup.pResultRows, pIte)) != NULL) {
3895
    SArray* pWins = (SArray*)(*pIte);
5
54liuyao 已提交
3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      SResultWindowInfo* pWin = (SResultWindowInfo*)taosArrayGet(pWins, i);
      pWin->pos.pageId = -1;
      pWin->pos.offset = -1;
    }
  }
  clearDiskbasedBuf(pInfo->streamAggSup.pResultBuf);
  cleanupResultRowInfo(&pInfo->binfo.resultRowInfo);
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
}

static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) {
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWin = taosArrayGet(pWins, i);
    taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition));
  }
}

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
  TSKEY                          maxTs = INT64_MIN;
  SExprSupp*                     pSup = &pOperator->exprSupp;
3921

5
54liuyao 已提交
3922 3923 3924
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
3925 3926 3927 3928 3929 3930 3931 3932 3933
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
    if (pBInfo->pRes->info.rows > 0) {
      printDataBlock(pBInfo->pRes, "Semi  Session");
      return pBInfo->pRes;
    }

    // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) {
      pInfo->returnDelete = true;
5
54liuyao 已提交
3934
      printDataBlock(pInfo->pDelRes, "Semi  Session");
5
54liuyao 已提交
3935 3936
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
3937 3938

    if (pInfo->pUpdateRes->info.rows > 0) {
5
54liuyao 已提交
3939 3940
      // process the rest of the data
      pOperator->status = OP_OPENED;
5
54liuyao 已提交
3941
      printDataBlock(pInfo->pUpdateRes, "Semi  Session");
5
54liuyao 已提交
3942 3943
      return pInfo->pUpdateRes;
    }
5
54liuyao 已提交
3944 3945 3946 3947
    // semi interval operator clear disk buffer
    clearStreamSessionOperator(pInfo);
    pOperator->status = OP_EXEC_DONE;
    return NULL;
5
54liuyao 已提交
3948 3949 3950 3951 3952 3953 3954 3955 3956
  }

  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  SOperatorInfo* downstream = pOperator->pDownstream[0];
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
3957
      clearSpecialDataBlock(pInfo->pUpdateRes);
5
54liuyao 已提交
3958 3959 3960
      break;
    }

5
54liuyao 已提交
3961
    if (pBlock->info.type == STREAM_CLEAR) {
5
54liuyao 已提交
3962
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
5
54liuyao 已提交
3963
      doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, 0, pSup->numOfExprs, 0, pWins);
5
54liuyao 已提交
3964 3965 3966 3967
      removeSessionResults(pStUpdated, pWins);
      taosArrayDestroy(pWins);
      copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
      break;
5
54liuyao 已提交
3968 3969
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      // gap must be 0
3970
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL);
5
54liuyao 已提交
3971 3972 3973
      copyDataBlock(pInfo->pDelRes, pBlock);
      pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
      break;
5
54liuyao 已提交
3974 3975 3976 3977 3978
    } else if (pBlock->info.type == STREAM_GET_ALL) {
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
      continue;
    }

5
54liuyao 已提交
3979 3980 3981 3982
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
5
54liuyao 已提交
3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, false);
    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
  // semi operator
  // closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
  //                    getResWinForSession);
5
54liuyao 已提交
3995
  copyUpdateResult(pStUpdated, pUpdated);
5
54liuyao 已提交
3996
  taosHashCleanup(pStUpdated);
5
54liuyao 已提交
3997

3998 3999
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4000
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
5
54liuyao 已提交
4001
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
4002 4003 4004 4005 4006 4007 4008 4009 4010 4011

  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
  if (pBInfo->pRes->info.rows > 0) {
    printDataBlock(pBInfo->pRes, "Semi  Session");
    return pBInfo->pRes;
  }

  // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) {
    pInfo->returnDelete = true;
5
54liuyao 已提交
4012
    printDataBlock(pInfo->pDelRes, "Semi  Session");
5
54liuyao 已提交
4013 4014
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
4015 4016

  if (pInfo->pUpdateRes->info.rows > 0) {
5
54liuyao 已提交
4017 4018
    // process the rest of the data
    pOperator->status = OP_OPENED;
5
54liuyao 已提交
4019
    printDataBlock(pInfo->pUpdateRes, "Semi  Session");
5
54liuyao 已提交
4020 4021
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
4022 4023 4024

  pOperator->status = OP_EXEC_DONE;
  return NULL;
5
54liuyao 已提交
4025
}
4026

4027 4028
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                       SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
4029 4030
  int32_t        code = TSDB_CODE_OUT_OF_MEMORY;
  SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo);
4031 4032 4033
  if (pOperator == NULL) {
    goto _error;
  }
4034
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
5
54liuyao 已提交
4035 4036 4037 4038 4039 4040 4041

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
    pInfo->isFinal = true;
    pOperator->name = "StreamSessionFinalAggOperator";
  } else {
    pInfo->isFinal = false;
    pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
5
54liuyao 已提交
4042
    pInfo->pUpdateRes->info.type = STREAM_CLEAR;
5
54liuyao 已提交
4043 4044 4045
    blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
    pOperator->name = "StreamSessionSemiAggOperator";
    pOperator->fpSet =
4046 4047
        createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL,
                            destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
4048 4049 4050 4051 4052
  }
  pOperator->operatorType = pPhyNode->type;
  if (numOfChild > 0) {
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
    for (int32_t i = 0; i < numOfChild; i++) {
4053
      SOperatorInfo* pChild = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
5
54liuyao 已提交
4054 4055 4056 4057
      if (pChild == NULL) {
        goto _error;
      }
      taosArrayPush(pInfo->pChildren, &pChild);
4058 4059 4060 4061 4062 4063
    }
  }
  return pOperator;

_error:
  if (pInfo != NULL) {
4064
    destroyStreamSessionAggOperatorInfo(pInfo, pOperator->exprSupp.numOfExprs);
4065 4066 4067 4068 4069 4070 4071
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5
54liuyao 已提交
4072 4073

void destroyStreamStateOperatorInfo(void* param, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
4074
  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;
4075
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
4076 4077 4078 4079 4080
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
4081
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
5
54liuyao 已提交
4082 4083 4084 4085 4086 4087
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
      destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput);
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
D
dapan1121 已提交
4088 4089

  taosMemoryFreeClear(param);
5
54liuyao 已提交
4090 4091 4092 4093 4094 4095 4096
}

int64_t getStateWinTsKey(void* data, int32_t index) {
  SStateWindowInfo* pStateWin = taosArrayGet(data, index);
  return pStateWin->winInfo.win.ekey;
}

X
Xiaoyu Wang 已提交
4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108
SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4109 4110 4111 4112 4113 4114 4115 4116
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayPush(pWinInfos, &win);
}

X
Xiaoyu Wang 已提交
4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128
SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayInsert(pWinInfos, index, &win);
}

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {
  if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) {
    return true;
  }
  return false;
}

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {
  return pKeyData && compareVal(pKeyData, &pWin->stateKey);
}

4148 4149 4150
SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
X
Xiaoyu Wang 已提交
4151 4152
  int32_t           size = taosArrayGetSize(pWinInfos);
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index + 1;
      return pWin;
    }
  }
  *pIndex = 0;
  return NULL;
}

4173 4174
SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
                                 SColumn* pCol, int32_t* pIndex) {
4175 4176
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
5
54liuyao 已提交
4177 4178 4179 4180 4181
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    *pIndex = 0;
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
X
Xiaoyu Wang 已提交
4182
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
  *pIndex = index + 1;
  return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol);
}

X
Xiaoyu Wang 已提交
4216 4217
int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, SColumnInfoData* pKeyCol, int32_t rows,
                              int32_t start, bool* allEqual, SHashObj* pSeDelete) {
5
54liuyao 已提交
4218 4219 4220 4221 4222
  *allEqual = true;
  SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex);
  for (int32_t i = start; i < rows; ++i) {
    char* pKeyData = colDataGetData(pKeyCol, i);
    if (!isTsInWindow(pWinInfo, pTs[i])) {
X
Xiaoyu Wang 已提交
4223
      if (isEqualStateKey(pWinInfo, pKeyData)) {
5
54liuyao 已提交
4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237
        int32_t size = taosArrayGetSize(pWinInfos);
        if (winIndex + 1 < size) {
          SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1);
          // ts belongs to the next window
          if (pTs[i] >= pNextWin->winInfo.win.skey) {
            return i - start;
          }
        }
      } else {
        return i - start;
      }
    }
    if (pWinInfo->winInfo.win.skey > pTs[i]) {
      if (pSeDelete && pWinInfo->winInfo.isOutput) {
X
Xiaoyu Wang 已提交
4238 4239
        taosHashPut(pSeDelete, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &pWinInfo->winInfo.win.skey,
                    sizeof(TSKEY));
5
54liuyao 已提交
4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251
        pWinInfo->winInfo.isOutput = false;
      }
      pWinInfo->winInfo.win.skey = pTs[i];
    }
    pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]);
    if (!isEqualStateKey(pWinInfo, pKeyData)) {
      *allEqual = false;
    }
  }
  return rows - start;
}

X
Xiaoyu Wang 已提交
4252 4253
static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int32_t tsIndex, SColumn* pCol,
                                int32_t keyIndex, SHashObj* pSeUpdated, SHashObj* pSeDeleted) {
5
54liuyao 已提交
4254 4255
  SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
  SColumnInfoData* pKeyColInfo = taosArrayGet(pBlock->pDataBlock, keyIndex);
X
Xiaoyu Wang 已提交
4256 4257 4258
  TSKEY*           tsCol = (TSKEY*)pTsColInfo->pData;
  bool             allEqual = false;
  int32_t          step = 1;
5
54liuyao 已提交
4259
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
4260 4261
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
4262
    SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], pBlock->info.groupId, &winIndex);
5
54liuyao 已提交
4263 4264 4265
    if (!pCurWin) {
      continue;
    }
4266
    step = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual,
X
Xiaoyu Wang 已提交
4267
                                 pSeDeleted);
5
54liuyao 已提交
4268 4269
    ASSERT(isTsInWindow(pCurWin, tsCol[i]) || isEqualStateKey(pCurWin, pKeyData));
    taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
4270
    deleteWindow(pAggSup->pCurWins, winIndex);
5
54liuyao 已提交
4271 4272 4273
  }
}

X
Xiaoyu Wang 已提交
4274 4275 4276
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated,
                                 SHashObj* pStDeleted) {
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
4277
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4278
  bool                         masterScan = true;
4279
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
X
Xiaoyu Wang 已提交
4280 4281 4282 4283 4284 4285 4286
  int64_t                      groupId = pSDataBlock->info.groupId;
  int64_t                      code = TSDB_CODE_SUCCESS;
  int32_t                      step = 1;
  bool                         ascScan = true;
  TSKEY*                       tsCols = NULL;
  SResultRow*                  pResult = NULL;
  int32_t                      winRows = 0;
5
54liuyao 已提交
4287
  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
4288 4289
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
4290
  } else {
X
Xiaoyu Wang 已提交
4291
    return;
5
54liuyao 已提交
4292
  }
4293
  
5
54liuyao 已提交
4294
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
4295
  blockDataEnsureCapacity(pAggSup->pScanBlock, pSDataBlock->info.rows);
X
Xiaoyu Wang 已提交
4296 4297
  SColumnInfoData*     pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
  for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
5
54liuyao 已提交
4298
    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
4299 4300 4301
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
4302 4303 4304
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
    bool              allEqual = true;
4305 4306 4307 4308
    SStateWindowInfo* pCurWin =
        getStateWindow(pAggSup, tsCols[i], pSDataBlock->info.groupId, pKeyData, &pInfo->stateCol, &winIndex);
    winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, pKeyColInfo, pSDataBlock->info.rows, i,
                                    &allEqual, pInfo->pSeDeleted);
5
54liuyao 已提交
4309
    if (!allEqual) {
4310 4311
      appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey,
          &pSDataBlock->info.groupId);
5
54liuyao 已提交
4312
      taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
4313
      deleteWindow(pAggSup->pCurWins, winIndex);
5
54liuyao 已提交
4314 4315
      continue;
    }
4316
    code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
4317 4318 4319 4320 4321
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    pCurWin->winInfo.isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
4322
      SWinRes value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId};
4323
      code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &value, sizeof(SWinRes));
5
54liuyao 已提交
4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336
      if (code != TSDB_CODE_SUCCESS) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
      pCurWin->winInfo.isOutput = true;
    }
  }
}

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

4337
  SExprSupp*                   pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4338
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4339
  SOptrBasicInfo*              pBInfo = &pInfo->binfo;
5
54liuyao 已提交
4340 4341 4342
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4343
      printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4344 4345
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
4346
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
4347
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
5
54liuyao 已提交
4348 4349
      doSetOperatorCompleted(pOperator);
    }
5
54liuyao 已提交
4350
    printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4351 4352 4353
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
4354 4355
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
4356
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
4357
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
4358 4359 4360 4361 4362
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4363
    printDataBlock(pBlock, "single state recv");
4364

5
54liuyao 已提交
4365
    if (pBlock->info.type == STREAM_CLEAR) {
X
Xiaoyu Wang 已提交
4366 4367
      doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
                          pSeUpdated, pInfo->pSeDeleted);
5
54liuyao 已提交
4368
      continue;
4369 4370 4371 4372 4373 4374
    } else if (pBlock->info.type == STREAM_DELETE_DATA) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins);
      copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
      taosArrayDestroy(pWins);
      continue;
4375
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4376
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState);
5
54liuyao 已提交
4377
      continue;
5
54liuyao 已提交
4378
    }
4379

5
54liuyao 已提交
4380 4381 4382 4383
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4384
    // the pDataBlock are always the same one, no need to call this again
4385
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4386 4387 4388 4389 4390
    doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
X
Xiaoyu Wang 已提交
4391

4392 4393
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState,
                     pInfo->ignoreExpiredData);
5
54liuyao 已提交
4394
  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
5
54liuyao 已提交
4395
  copyUpdateResult(pSeUpdated, pUpdated);
5
54liuyao 已提交
4396 4397
  taosHashCleanup(pSeUpdated);

4398 4399
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4400 4401 4402 4403
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4404
    printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4405 4406
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
4407
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
4408
  printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4409 4410 4411
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

4412 4413 4414 4415
int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo));
}

X
Xiaoyu Wang 已提交
4416 4417 4418 4419 4420 4421
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
  SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
4422
  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
4423

X
Xiaoyu Wang 已提交
4424 4425
  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
4426 4427 4428 4429
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

4430 4431
  SExprSupp* pSup = &pOperator->exprSupp;

X
Xiaoyu Wang 已提交
4432
  int32_t    numOfCols = 0;
5
54liuyao 已提交
4433 4434 4435
  SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
4436
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
4437 4438 4439 4440 4441 4442 4443 4444 4445
  if (pStateNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

4446
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
X
Xiaoyu Wang 已提交
4447 4448
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
5
54liuyao 已提交
4449 4450
      .calTrigger = pStateNode->window.triggerType,
      .maxTs = INT64_MIN,
X
Xiaoyu Wang 已提交
4451
  };
5
54liuyao 已提交
4452
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
4453

4454
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
4455 4456 4457 4458
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

4459
  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
4460 4461 4462 4463 4464 4465 4466 4467 4468
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }

4469
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
4470 4471 4472 4473 4474
  pInfo->primaryTsIndex = tsSlotId;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
4475
  // pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
4476 4477
  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete
5
54liuyao 已提交
4478
  pInfo->pChildren = NULL;
5
54liuyao 已提交
4479
  pInfo->ignoreExpiredData = pStateNode->window.igExpired;
5
54liuyao 已提交
4480 4481

  pOperator->name = "StreamStateAggOperator";
4482
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
5
54liuyao 已提交
4483 4484
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
4485 4486
  pOperator->exprSupp.numOfExprs = numOfCols;
  pOperator->exprSupp.pExprInfo = pExprInfo;
5
54liuyao 已提交
4487 4488
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
4489 4490 4491
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL,
                                         destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType);
5
54liuyao 已提交
4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
  return pOperator;

_error:
  destroyStreamStateOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
4505

4506 4507
void destroyMergeAlignedIntervalOperatorInfo(void* param, int32_t numOfOutput) {
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param;
D
dapan1121 已提交
4508 4509
  destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo, numOfOutput);

D
dapan1121 已提交
4510
  taosMemoryFreeClear(param);
4511 4512
}

4513 4514
static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId,
                                                SSDataBlock* pResultBlock, TSKEY wstartTs) {
4515
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
D
dapan1121 已提交
4516
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
4517
  SExecTaskInfo*                        pTaskInfo = pOperatorInfo->pTaskInfo;
4518 4519

  SExprSupp* pSup = &pOperatorInfo->exprSupp;
4520
  bool       ascScan = (iaInfo->order == TSDB_ORDER_ASC);
4521

4522 4523 4524 4525
  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId);
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf,
                                                            GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
  ASSERT(p1 != NULL);
4526

4527 4528
  finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pSup->pCtx, pSup->pExprInfo, pSup->numOfExprs,
                                       pSup->rowEntryInfoOffset, pResultBlock, pTaskInfo);
4529
  taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
4530

4531
  return 0;
4532 4533
}

4534 4535
static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
                                          SSDataBlock* pBlock, int32_t scanFlag, SSDataBlock* pResultBlock) {
4536
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
D
dapan1121 已提交
4537
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
4538 4539

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
4540
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
4541 4542

  int32_t     startPos = 0;
4543
  int32_t     numOfOutput = pSup->numOfExprs;
4544
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
4545 4546 4547 4548
  uint64_t    tableGroupId = pBlock->info.groupId;
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

4549 4550
  STimeWindow win;
  win.skey = blockStartTs;
4551 4552
  win.ekey =
      taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1;
4553

4554
  // TODO: remove the hash table (groupid + winkey => result row position)
4555 4556
  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
4557 4558 4559 4560
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
  }

4561 4562
  TSKEY       currTs = blockStartTs;
  TSKEY       currPos = startPos;
4563
  STimeWindow currWin = win;
4564
  while (1) {
4565 4566
    ++currPos;
    if (currPos >= pBlock->info.rows) {
4567 4568
      break;
    }
4569 4570 4571 4572
    if (tsCols[currPos] == currTs) {
      continue;
    } else {
      updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
4573 4574
      doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
                       tsCols, pBlock->info.rows, numOfOutput, iaInfo->order);
4575

4576
      outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs);
4577 4578 4579

      currTs = tsCols[currPos];
      currWin.skey = currTs;
4580
      currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit,
4581 4582
                                 iaInfo->interval.precision) -
                     1;
4583
      startPos = currPos;
4584 4585
      ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                   pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
4586 4587 4588
      if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
4589 4590
    }
  }
4591
  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
4592 4593
  doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
                   tsCols, pBlock->info.rows, numOfOutput, iaInfo->order);
4594

4595
  outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs);
4596 4597
}

4598
static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
S
shenglian zhou 已提交
4599
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
4600

4601
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperator->info;
D
dapan1121 已提交
4602
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
4603 4604 4605 4606
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

4607
  SExprSupp*   pSup = &pOperator->exprSupp;
4608
  SSDataBlock* pRes = iaInfo->binfo.pRes;
4609
  blockDataCleanup(pRes);
4610
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);
4611

4612 4613 4614 4615 4616 4617 4618 4619 4620 4621
  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
        miaInfo->groupId = pBlock->info.groupId;
4622
        miaInfo->prefetchedBlock = NULL;
4623
      }
4624

4625 4626 4627 4628
      if (pBlock == NULL) {
        miaInfo->inputBlocksFinished = true;
        break;
      }
4629

4630 4631 4632 4633 4634 4635 4636
      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
        miaInfo->groupId = pBlock->info.groupId;
      } else if (miaInfo->groupId != pBlock->info.groupId) {
        miaInfo->prefetchedBlock = pBlock;
        break;
      }
4637

4638
      getTableScanInfo(pOperator, &iaInfo->order, &scanFlag);
4639
      setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->order, scanFlag, true);
4640
      doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);
4641
      doFilter(miaInfo->pCondition, pRes, NULL);
4642
      if (pRes->info.rows >= pOperator->resultInfo.capacity) {
4643 4644 4645 4646 4647
        break;
      }
    }

    pRes->info.groupId = miaInfo->groupId;
4648
  }
4649
  miaInfo->hasGroupId = false;
4650

4651
  if (miaInfo->inputBlocksFinished) {
4652 4653 4654 4655 4656 4657 4658 4659
    doSetOperatorCompleted(pOperator);
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

4660 4661
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo,
                                                      int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval,
4662 4663
                                                      int32_t primaryTsSlotId, SNode* pCondition,
                                                      SExecTaskInfo* pTaskInfo) {
4664
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo));
4665
  SOperatorInfo*                        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
4666 4667 4668 4669
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

D
dapan1121 已提交
4670 4671 4672 4673 4674 4675
  miaInfo->intervalAggOperatorInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
  if (miaInfo->intervalAggOperatorInfo == NULL) {
    goto _error;
  }

  SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
4676
  SExprSupp*                pSup = &pOperator->exprSupp;
4677

4678
  miaInfo->pCondition = pCondition;
4679 4680 4681
  iaInfo->win = pTaskInfo->window;
  iaInfo->order = TSDB_ORDER_ASC;
  iaInfo->interval = *pInterval;
4682 4683
  iaInfo->execModel = pTaskInfo->execModel;
  iaInfo->primaryTsIndex = primaryTsSlotId;
4684 4685

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
4686
  initResultSizeInfo(&pOperator->resultInfo, 4096);
4687

4688 4689
  int32_t code =
      initAggInfo(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
4690
  initBasicInfo(&iaInfo->binfo, pResBlock);
4691

4692
  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
4693

4694
  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, iaInfo);
4695 4696
  if (iaInfo->timeWindowInterpo) {
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
4697 4698
  }

4699
  if (code != TSDB_CODE_SUCCESS) {
4700 4701 4702
    goto _error;
  }

4703
  initResultRowInfo(&iaInfo->binfo.resultRowInfo);
4704

4705 4706
  pOperator->name = "TimeMergeAlignedIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL;
4707 4708
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
4709
  pOperator->exprSupp.pExprInfo = pExprInfo;
4710
  pOperator->pTaskInfo = pTaskInfo;
4711
  pOperator->exprSupp.numOfExprs = numOfCols;
4712
  pOperator->info = miaInfo;
4713

4714 4715
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeAlignedIntervalAgg, NULL, NULL,
                                         destroyMergeAlignedIntervalOperatorInfo, NULL, NULL, NULL);
4716 4717 4718 4719 4720 4721 4722 4723 4724

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
4725
  destroyMergeAlignedIntervalOperatorInfo(miaInfo, numOfCols);
4726
  taosMemoryFreeClear(miaInfo);
4727 4728 4729 4730
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
4731 4732 4733 4734 4735

//=====================================================================================================================
// merge interval operator
typedef struct SMergeIntervalAggOperatorInfo {
  SIntervalAggOperatorInfo intervalAggOperatorInfo;
L
Liu Jicong 已提交
4736 4737 4738 4739 4740 4741
  SList*                   groupIntervals;
  SListIter                groupIntervalsIter;
  bool                     hasGroupId;
  uint64_t                 groupId;
  SSDataBlock*             prefetchedBlock;
  bool                     inputBlocksFinished;
4742 4743
} SMergeIntervalAggOperatorInfo;

S
slzhou 已提交
4744
typedef struct SGroupTimeWindow {
L
Liu Jicong 已提交
4745
  uint64_t    groupId;
S
slzhou 已提交
4746 4747 4748
  STimeWindow window;
} SGroupTimeWindow;

4749 4750
void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) {
  SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
S
slzhou 已提交
4751
  tdListFree(miaInfo->groupIntervals);
4752
  destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput);
4753

D
dapan1121 已提交
4754
  taosMemoryFreeClear(param);
4755 4756
}

L
Liu Jicong 已提交
4757 4758
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win,
                                    SSDataBlock* pResultBlock) {
4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
  bool                           ascScan = (iaInfo->order == TSDB_ORDER_ASC);
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId);
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf,
                                                            GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
  ASSERT(p1 != NULL);
  finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo,
                                       pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo);
  taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
  return TSDB_CODE_SUCCESS;
}

4775 4776 4777 4778 4779 4780 4781 4782
static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock,
                                        STimeWindow* newWin) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
  bool                           ascScan = (iaInfo->order == TSDB_ORDER_ASC);
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

S
slzhou 已提交
4783 4784
  SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin};
  tdListAppend(miaInfo->groupIntervals, &groupTimeWindow);
4785

S
slzhou 已提交
4786 4787 4788 4789 4790
  SListIter iter = {0};
  tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD);
  SListNode* listNode = NULL;
  while ((listNode = tdListNext(&iter)) != NULL) {
    SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
L
Liu Jicong 已提交
4791
    if (prevGrpWin->groupId != tableGroupId) {
S
slzhou 已提交
4792 4793 4794
      continue;
    }
    STimeWindow* prevWin = &prevGrpWin->window;
H
Haojun Liao 已提交
4795
    if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) {
S
slzhou 已提交
4796 4797 4798
      finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
      tdListPopNode(miaInfo->groupIntervals, listNode);
    }
4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819
  }

  return 0;
}

static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                                   int32_t scanFlag, SSDataBlock* pResultBlock) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*     pExprSup = &pOperatorInfo->exprSupp;

  int32_t     startPos = 0;
  int32_t     numOfOutput = pExprSup->numOfExprs;
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
  bool        ascScan = (iaInfo->order == TSDB_ORDER_ASC);
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

X
Xiaoyu Wang 已提交
4820 4821
  STimeWindow win =
      getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval, iaInfo->order);
4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924

  int32_t ret =
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                             numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
  }

  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->order);
  ASSERT(forwardRows > 0);

  // prev time window not interpolation yet.
  if (iaInfo->timeWindowInterpo) {
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult);
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);

    // restore current time window
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                                 numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    // window start key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup);
  }

  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true);
  doApplyFunctions(pTaskInfo, pExprSup->pCtx, &win, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
                   pBlock->info.rows, numOfOutput, iaInfo->order);
  doCloseWindow(pResultRowInfo, iaInfo, pResult);

  // output previous interval results after this interval (&win) is closed
  outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win);

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = forwardRows - 1 + startPos;
    startPos = getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->order);
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
    int32_t code =
        setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                               pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    ekey = ascScan ? nextWin.ekey : nextWin.skey;
    forwardRows =
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->order);

    // window start(end) key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup);

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true);
    doApplyFunctions(pTaskInfo, pExprSup->pCtx, &nextWin, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     tsCols, pBlock->info.rows, numOfOutput, iaInfo->order);
    doCloseWindow(pResultRowInfo, iaInfo, pResult);

    // output previous interval results after this interval (&nextWin) is closed
    outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin);
  }

  if (iaInfo->timeWindowInterpo) {
    saveDataBlockLastRow(iaInfo->pPrevValues, pBlock, iaInfo->pInterpCols);
  }
}

static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExprSupp*                     pExpSupp = &pOperator->exprSupp;

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);

  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
        miaInfo->groupId = pBlock->info.groupId;
        miaInfo->prefetchedBlock = NULL;
      }

      if (pBlock == NULL) {
S
slzhou 已提交
4925
        tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD);
4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947
        miaInfo->inputBlocksFinished = true;
        break;
      }

      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
        miaInfo->groupId = pBlock->info.groupId;
      } else if (miaInfo->groupId != pBlock->info.groupId) {
        miaInfo->prefetchedBlock = pBlock;
        break;
      }

      getTableScanInfo(pOperator, &iaInfo->order, &scanFlag);
      setInputDataBlock(pOperator, pExpSupp->pCtx, pBlock, iaInfo->order, scanFlag, true);
      doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);

      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
        break;
      }
    }

    pRes->info.groupId = miaInfo->groupId;
4948 4949 4950
  }

  if (miaInfo->inputBlocksFinished) {
S
slzhou 已提交
4951
    SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter);
4952

S
slzhou 已提交
4953 4954 4955 4956
    if (listNode != NULL) {
      SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data);
      finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
      pRes->info.groupId = grpWin->groupId;
4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977
    }
  }

  if (pRes->info.rows == 0) {
    doSetOperatorCompleted(pOperator);
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                               SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
                                               SExecTaskInfo* pTaskInfo) {
  SMergeIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

S
slzhou 已提交
4978
  miaInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow));
4979

4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991
  SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;

  iaInfo->win = pTaskInfo->window;
  iaInfo->order = TSDB_ORDER_ASC;
  iaInfo->interval = *pInterval;
  iaInfo->execModel = pTaskInfo->execModel;

  iaInfo->primaryTsIndex = primaryTsSlotId;

  SExprSupp* pExprSupp = &pOperator->exprSupp;

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
4992
  initResultSizeInfo(&pOperator->resultInfo, 4096);
4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033

  int32_t code = initAggInfo(pExprSupp, &iaInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&iaInfo->binfo, pResBlock);

  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);

  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, numOfCols, iaInfo);
  if (iaInfo->timeWindowInterpo) {
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
    if (iaInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
  }

  initResultRowInfo(&iaInfo->binfo.resultRowInfo);

  pOperator->name = "TimeMergeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL;
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->exprSupp.pExprInfo = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
  pOperator->info = miaInfo;

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL,
                                         destroyMergeIntervalOperatorInfo, NULL, NULL, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
  destroyMergeIntervalOperatorInfo(miaInfo, numOfCols);
  taosMemoryFreeClear(miaInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
5034
}