timewindowoperator.c 231.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15
#include "executorimpl.h"
X
Xiaoyu Wang 已提交
16
#include "function.h"
5
54liuyao 已提交
17
#include "functionMgt.h"
H
Haojun Liao 已提交
18
#include "tcommon.h"
5
54liuyao 已提交
19
#include "tcompare.h"
L
Liu Jicong 已提交
20
#include "tdatablock.h"
H
Haojun Liao 已提交
21
#include "tfill.h"
22
#include "ttime.h"
23 24 25 26 27 28

typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;

5
54liuyao 已提交
29 30
#define IS_FINAL_OP(op) ((op)->isFinal)

5
54liuyao 已提交
31 32
typedef struct SPullWindowInfo {
  STimeWindow window;
L
Liu Jicong 已提交
33
  uint64_t    groupId;
5
54liuyao 已提交
34 35
} SPullWindowInfo;

36 37
typedef struct SOpenWindowInfo {
  SResultRowPosition pos;
L
Liu Jicong 已提交
38
  uint64_t           groupId;
39 40
} SOpenWindowInfo;

5
54liuyao 已提交
41
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
5
54liuyao 已提交
42

43 44
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);

L
Liu Jicong 已提交
45 46
static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult,
                                              uint64_t groupId);
47 48
static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult);

X
Xiaoyu Wang 已提交
49
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }
50 51 52

static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
53
                                      int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
54 55 56 57 58 59 60 61 62 63 64
                                      SExecTaskInfo* pTaskInfo) {
  SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                  masterscan, tableGroupId, pTaskInfo, true, pAggSup);

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // set time window for current result
  pResultRow->win = (*win);
65

66
  *pResult = pResultRow;
67
  setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
68

69 70 71 72 73 74 75 76 77 78 79 80 81
  return TSDB_CODE_SUCCESS;
}

static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) {
  int64_t* ts = (int64_t*)pColData->pData;
  int32_t  delta = includeEndpoint ? 1 : 0;

  int64_t duration = pWin->ekey - pWin->skey + delta;
  ts[2] = duration;            // set the duration
  ts[3] = pWin->skey;          // window start key
  ts[4] = pWin->ekey + delta;  // window end key
}

82
static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) {
83 84 85
  pRowSup->win.ekey = ts;
  pRowSup->prevTs = ts;
  pRowSup->numOfRows += 1;
86
  pRowSup->groupId = groupId;
87 88
}

dengyihao's avatar
dengyihao 已提交
89 90
static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex,
                                     uint64_t groupId) {
91 92 93
  pRowSup->startRowIndex = rowIndex;
  pRowSup->numOfRows = 0;
  pRowSup->win.skey = tsList[rowIndex];
94
  pRowSup->groupId = groupId;
95 96 97 98
}

static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
                                                   int16_t pos, int16_t order, int64_t* pData) {
99
  int32_t forwardRows = 0;
100 101 102 103

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
104
      forwardRows = end;
105 106

      if (pData[end + pos] == ekey) {
107
        forwardRows += 1;
108 109 110
      }
    }
  } else {
111
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
112
    if (end >= 0) {
113
      forwardRows = end;
114

115
      if (pData[end + pos] == ekey) {
116
        forwardRows += 1;
117 118
      }
    }
X
Xiaoyu Wang 已提交
119 120 121 122 123 124 125 126
    //    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
    //    if (end >= 0) {
    //      forwardRows = pos - end;
    //
    //      if (pData[end] == ekey) {
    //        forwardRows += 1;
    //      }
    //    }
127 128
  }

129 130
  assert(forwardRows >= 0);
  return forwardRows;
131 132
}

5
54liuyao 已提交
133
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY*  keyList = (TSKEY*)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
150 151 152 153 154 155 156 157 158 159 160
      if (key >= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key < keyList[lastPos]) {
        lastPos += 1;
        if (lastPos >= num) {
          return -1;
        } else {
          return lastPos;
        }
      }
161 162 163 164 165 166

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        firstPos = midPos + 1;
167 168
      } else if (key > keyList[midPos]) {
        lastPos = midPos - 1;
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

X
Xiaoyu Wang 已提交
204 205
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
                                 __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num = -1;
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);

  if (order == TSDB_ORDER_ASC) {
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
      if (item != NULL) {
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
227
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
228 229
      }
    } else {
230
      num = pDataBlockInfo->rows - startPos;
231
      if (item != NULL) {
232
        item->lastKey = pDataBlockInfo->window.ekey + step;
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
      }
    }
  }

  assert(num >= 0);
  return num;
}

static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
    return;
  }

  int64_t key = tw->skey, interval = pInterval->interval;
  // convert key to second
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;

  if (pInterval->intervalUnit == 'y') {
    interval *= 12;
  }

  struct tm tm;
  time_t    t = (time_t)key;
  taosLocalTime(&t, &tm);

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
264
  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
265 266 267 268

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
269
  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
270 271 272 273

  tw->ekey -= 1;
}

274 275
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs,
                               int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
276
  SqlFunctionCtx* pCtx = pSup->pCtx;
277

278
  int32_t index = 1;
279
  for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
H
Haojun Liao 已提交
280
    if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
281 282 283 284
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

X
Xiaoyu Wang 已提交
285
    SFunctParam*     pParam = &pCtx[k].param[0];
286 287
    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId);

288
    ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey);
289

290
    double v1 = 0, v2 = 0, v = 0;
291
    if (prevRowIndex == -1) {
292
      SGroupKeys* p = taosArrayGet(pPrevValues, index);
293
      GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData);
294
    } else {
295
      GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex));
296 297
    }

298
    GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
299

300
#if 0
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            //            pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
320 321
#endif

X
Xiaoyu Wang 已提交
322 323 324
    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
    SPoint point = (SPoint){.key = windowKey, .val = &v};
325

X
Xiaoyu Wang 已提交
326
    taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
327

X
Xiaoyu Wang 已提交
328 329 330 331 332 333
    if (type == RESULT_ROW_START_INTERP) {
      pCtx[k].start.key = point.key;
      pCtx[k].start.val = v;
    } else {
      pCtx[k].end.key = point.key;
      pCtx[k].end.val = v;
334
    }
X
Xiaoyu Wang 已提交
335 336 337

    index += 1;
  }
338
#if 0
339
  }
340
#endif
341 342 343 344 345 346 347 348 349 350 351 352 353 354
}

static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

355 356
static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock,
                                              const TSKEY* tsCols, STimeWindow* win, SExprSupp* pSup) {
357
  bool ascQuery = (pInfo->inputOrder == TSDB_ORDER_ASC);
358

359
  TSKEY curTs = tsCols[pos];
360 361

  SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
X
Xiaoyu Wang 已提交
362
  TSKEY       lastTs = *(int64_t*)pTsKey->pData;
363 364 365 366 367

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery ? win->skey : win->ekey;
  if (key == curTs) {
368
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
369 370 371
    return true;
  }

372 373
  // it is the first time window, no need to do interpolation
  if (pTsKey->isNull && pos == 0) {
374
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
375 376
  } else {
    TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]);
377 378
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key,
                              RESULT_ROW_START_INTERP, pSup);
379 380 381 382 383
  }

  return true;
}

384 385 386
static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex,
                                            SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey,
                                            STimeWindow* win) {
387
  int32_t order = pInfo->inputOrder;
388 389

  TSKEY actualEndKey = tsCols[endRowIndex];
390
  TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey;
391 392

  // not ended in current data block, do not invoke interpolation
393
  if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) {
394
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
395 396 397
    return false;
  }

398
  // there is actual end point of current time window, no interpolation needs
399
  if (key == actualEndKey) {
400
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
401 402 403
    return true;
  }

404
  int32_t nextRowIndex = endRowIndex + 1;
405 406 407
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
408 409
  doTimeWindowInterpolation(pInfo->pPrevValues, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key,
                            RESULT_ROW_END_INTERP, pSup);
410 411 412
  return true;
}

5
54liuyao 已提交
413
bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo) {
414 415
  if (pInterval->interval != pInterval->sliding &&
      (pWin->ekey < pBlockInfo->calWin.skey || pWin->skey > pBlockInfo->calWin.ekey)) {
5
54liuyao 已提交
416 417 418 419 420
    return false;
  }
  return true;
}

421
static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
5
54liuyao 已提交
422
                                      TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
X
Xiaoyu Wang 已提交
423
  bool ascQuery = (order == TSDB_ORDER_ASC);
424 425 426 427 428 429 430 431 432 433

  int32_t precision = pInterval->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
    return -1;
  }

5
54liuyao 已提交
434 435 436 437
  if (!inSlidingWindow(pInterval, pNext, pDataBlockInfo) && order == TSDB_ORDER_ASC) {
    return -1;
  }

438
  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
439 440 441 442
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
443
    startPos = prevPosition + 1;
444
  } else {
445
    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
446 447
      startPos = 0;
    } else {
448
      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
    }
  }

  /* interp query with fill should not skip time window */
  //  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
  //    return startPos;
  //  }

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
    if (ascQuery) {
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
      }
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
      }
    }
  }

  return startPos;
}

492 493
static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
  ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp == true;
  }
}

static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp = true;
  }
}

510 511
static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult,
                                        STimeWindow* win, int32_t startPos, int32_t forwardRows, SExprSupp* pSup) {
512
  if (!pInfo->timeWindowInterpo) {
513 514 515
    return;
  }

516
  ASSERT(pBlock != NULL);
517 518 519 520 521
  if (pBlock->pDataBlock == NULL) {
    //    tscError("pBlock->pDataBlock == NULL");
    return;
  }

522
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
523 524

  TSKEY* tsCols = (TSKEY*)(pColInfo->pData);
525
  bool   done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
526
  if (!done) {  // it is not interpolated, now start to generated the interpolated value
527
    bool interp = setTimeWindowInterpolationStartTs(pInfo, startPos, pBlock, tsCols, win, pSup);
528 529 530 531
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
532
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
533 534 535 536 537 538 539 540
  }

  // point interpolation does not require the end key time window interpolation.
  //  if (pointInterpQuery) {
  //    return;
  //  }

  // interpolation query does not generate the time window end interpolation
541
  done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
542
  if (!done) {
543
    int32_t endRowIndex = startPos + forwardRows - 1;
544

545
    TSKEY endKey = (pInfo->inputOrder == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
546
    bool  interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
547 548 549 550
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
551
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
552 553 554
  }
}

555 556
static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) {
  if (pBlock->pDataBlock == NULL) {
557 558 559
    return;
  }

560 561 562 563 564 565 566
  size_t num = taosArrayGetSize(pPrevKeys);
  for (int32_t k = 0; k < num; ++k) {
    SColumn* pc = taosArrayGet(pCols, k);

    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId);

    SGroupKeys* pkey = taosArrayGet(pPrevKeys, k);
X
Xiaoyu Wang 已提交
567
    for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) {
568 569 570 571 572 573 574 575 576 577 578 579 580 581
      if (colDataIsNull_s(pColInfo, i)) {
        continue;
      }

      char* val = colDataGetData(pColInfo, i);
      if (IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, varDataTLen(val));
        ASSERT(varDataTLen(val) <= pkey->bytes);
      } else {
        memcpy(pkey->pData, val, pkey->bytes);
      }

      break;
    }
582 583 584
  }
}

585 586 587 588
static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo,
                                       SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) {
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

589
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
590
  SExprSupp*                pSup = &pOperatorInfo->exprSupp;
591

L
Liu Jicong 已提交
592 593
  int32_t startPos = 0;
  int32_t numOfOutput = pSup->numOfExprs;
594

595
  SResultRow* pResult = NULL;
596

597
  while (1) {
L
Liu Jicong 已提交
598 599 600
    SListNode*          pn = tdListGetHead(pResultRowInfo->openWindow);
    SOpenWindowInfo*    pOpenWin = (SOpenWindowInfo*)pn->data;
    uint64_t            groupId = pOpenWin->groupId;
601
    SResultRowPosition* p1 = &pOpenWin->pos;
602 603 604
    if (p->pageId == p1->pageId && p->offset == p1->offset) {
      break;
    }
605

606
    SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1, false);
607
    ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
608

609
    if (pr->closed) {
X
Xiaoyu Wang 已提交
610 611
      ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
             isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
612 613
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
614 615
      continue;
    }
616

617
    STimeWindow w = pr->win;
618 619
    int32_t     ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx,
                                             numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
620
    if (ret != TSDB_CODE_SUCCESS) {
621
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
622 623 624 625
    }

    ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP));

X
Xiaoyu Wang 已提交
626 627
    SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
    int64_t     prevTs = *(int64_t*)pTsKey->pData;
628 629 630 631
    if (groupId == pBlock->info.groupId) {
      doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey,
                                RESULT_ROW_END_INTERP, pSup);
    }
632 633

    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
634
    setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
635

636
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, true);
H
Haojun Liao 已提交
637 638
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, pBlock->info.rows,
                     numOfExprs);
639 640 641

    if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
      closeResultRow(pr);
642 643
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
X
Xiaoyu Wang 已提交
644
    } else {  // the remains are can not be closed yet.
645
      break;
646
    }
647
  }
648
}
649

5
54liuyao 已提交
650
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
5
54liuyao 已提交
651
  if (!pBlock || pBlock->info.rows == 0) {
5
54liuyao 已提交
652
    qDebug("===stream===printDataBlock: Block is Null or Empty");
5
54liuyao 已提交
653 654
    return;
  }
L
Liu Jicong 已提交
655
  char* pBuf = NULL;
5
54liuyao 已提交
656 657
  qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
  taosMemoryFree(pBuf);
5
54liuyao 已提交
658 659
}

5
54liuyao 已提交
660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715
typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);

int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;
      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;
      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;
      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

      if (comparefn(pKey, keyList, lastPos) > 0) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

5
54liuyao 已提交
716
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
717

X
Xiaoyu Wang 已提交
718 719 720
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;
5
54liuyao 已提交
721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (key >= getValuefn(keyList, lastPos)) return lastPos;
      if (key == getValuefn(keyList, firstPos)) return firstPos;
      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (key <= getValuefn(keyList, firstPos)) return firstPos;
      if (key == getValuefn(keyList, lastPos)) return lastPos;

      if (key > getValuefn(keyList, lastPos)) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
767 768
  }

5
54liuyao 已提交
769 770 771
  return midPos;
}

5
54liuyao 已提交
772
int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {
L
Liu Jicong 已提交
773
  SArray*          res = (SArray*)data;
5
54liuyao 已提交
774
  SPullWindowInfo* pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
775
  SPullWindowInfo* pData = (SPullWindowInfo*)pKey;
5
54liuyao 已提交
776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806
  if (pData->window.skey == pos->window.skey) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->window.skey > pos->window.skey) {
    return 1;
  }
  return -1;
}

static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {
  int32_t size = taosArrayGetSize(pPullWins);
  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);
  if (index == -1) {
    index = 0;
  } else {
    if (comparePullWinKey(pPullInfo, pPullWins, index) > 0) {
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
807 808 809
int32_t compareResKey(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
  SResKeyPos* pos = taosArrayGetP(res, index);
H
Haojun Liao 已提交
810
  SWinKey*    pData = (SWinKey*)pKey;
5
54liuyao 已提交
811 812 813 814 815 816 817 818 819 820 821 822 823
  if (pData->ts == *(int64_t*)pos->key) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->ts > *(int64_t*)pos->key) {
    return 1;
  }
  return -1;
}

824
static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SArray* pUpdated) {
5
54liuyao 已提交
825
  int32_t size = taosArrayGetSize(pUpdated);
H
Haojun Liao 已提交
826
  SWinKey data = {.ts = ts, .groupId = groupId};
5
54liuyao 已提交
827
  int32_t index = binarySearchCom(pUpdated, size, &data, TSDB_ORDER_DESC, compareResKey);
5
54liuyao 已提交
828 829 830
  if (index == -1) {
    index = 0;
  } else {
5
54liuyao 已提交
831
    if (compareResKey(&data, pUpdated, index) > 0) {
5
54liuyao 已提交
832 833 834 835 836
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
H
Haojun Liao 已提交
837

5
54liuyao 已提交
838 839 840 841 842
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
5
54liuyao 已提交
843 844
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
X
Xiaoyu Wang 已提交
845
  if (taosArrayInsert(pUpdated, index, &newPos) == NULL) {
5
54liuyao 已提交
846 847 848 849 850
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
851 852 853 854 855 856 857 858
static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SHashObj* pUpdatedMap) {
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
H
Haojun Liao 已提交
859 860
  SWinKey key = {.ts = ts, .groupId = groupId};
  if (taosHashPut(pUpdatedMap, &key, sizeof(SWinKey), &newPos, sizeof(void*)) != TSDB_CODE_SUCCESS) {
5
54liuyao 已提交
861 862 863
    taosMemoryFree(newPos);
  }
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
864 865
}

5
54liuyao 已提交
866
static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj* pUpdatedMap) {
dengyihao's avatar
dengyihao 已提交
867
  return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap);
5
54liuyao 已提交
868 869
}

5
54liuyao 已提交
870 871 872 873
static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SHashObj* pUpdatedMap) {
  return saveWinResult(ts, -1, -1, groupId, pUpdatedMap);
}

5
54liuyao 已提交
874 875
static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
  return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated);
5
54liuyao 已提交
876 877
}

5
54liuyao 已提交
878
static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) {
5
54liuyao 已提交
879 880
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
H
Haojun Liao 已提交
881 882
    SWinKey* pW = taosArrayGet(pWins, i);
    taosHashRemove(pUpdatedMap, pW, sizeof(SWinKey));
5
54liuyao 已提交
883 884 885
  }
}

886
int64_t getWinReskey(void* data, int32_t index) {
887
  SArray*  res = (SArray*)data;
H
Haojun Liao 已提交
888
  SWinKey* pos = taosArrayGet(res, index);
889 890 891
  return pos->ts;
}

5
54liuyao 已提交
892 893
int32_t compareWinRes(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
894
  SWinKey*    pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
895
  SResKeyPos* pData = (SResKeyPos*)pKey;
5
54liuyao 已提交
896 897 898 899 900 901 902 903 904 905 906 907 908
  if (*(int64_t*)pData->key == pos->ts) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (*(int64_t*)pData->key > pos->ts) {
    return 1;
  }
  return -1;
}

5
54liuyao 已提交
909
static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) {
L
Liu Jicong 已提交
910 911
  int32_t delSize = taosArrayGetSize(pDelWins);
  if (taosHashGetSize(pUpdatedMap) == 0 || delSize == 0) {
5
54liuyao 已提交
912
    return;
dengyihao's avatar
dengyihao 已提交
913
  }
L
Liu Jicong 已提交
914
  void* pIte = NULL;
5
54liuyao 已提交
915
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
916
    SResKeyPos* pResKey = *(SResKeyPos**)pIte;
5
54liuyao 已提交
917 918
    int32_t     index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes);
    if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) {
919
      taosArrayRemove(pDelWins, index);
920
      delSize = taosArrayGetSize(pDelWins);
921 922 923 924
    }
  }
}

5
54liuyao 已提交
925 926 927
bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) {
  ASSERT(pTwSup->maxTs == INT64_MIN || pTwSup->maxTs > 0);
  return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark;
5
54liuyao 已提交
928 929
}

5
54liuyao 已提交
930 931 932 933 934
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); }

bool needDeleteWindowBuf(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) {
  return pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark;
}
5
54liuyao 已提交
935

5
54liuyao 已提交
936
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
937
                            int32_t scanFlag) {
938
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
939

940
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
941
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
942

X
Xiaoyu Wang 已提交
943
  int32_t     startPos = 0;
944
  int32_t     numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
945 946
  int64_t*    tsCols = extractTsCol(pBlock, pInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
947
  bool        ascScan = (pInfo->inputOrder == TSDB_ORDER_ASC);
X
Xiaoyu Wang 已提交
948 949
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;
950

951 952
  STimeWindow win =
      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder);
953 954
  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
955 956
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
957
  }
X
Xiaoyu Wang 已提交
958 959
  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
960
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
961
  ASSERT(forwardRows > 0);
962 963

  // prev time window not interpolation yet.
964
  if (pInfo->timeWindowInterpo) {
965
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
966
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);
967 968

    // restore current time window
969 970
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
971
    if (ret != TSDB_CODE_SUCCESS) {
972
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
973 974
    }

975
    // window start key interpolation
976
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
977
  }
978

979 980
  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
981
                   numOfOutput);
982 983

  doCloseWindow(pResultRowInfo, pInfo, pResult);
984 985 986

  STimeWindow nextWin = win;
  while (1) {
987
    int32_t prevEndPos = forwardRows - 1 + startPos;
988
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->inputOrder);
989 990 991 992
    if (startPos < 0) {
      break;
    }
    // null data, failed to allocate more memory buffer
X
Xiaoyu Wang 已提交
993
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
994
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
995
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
996
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
997 998
    }

X
Xiaoyu Wang 已提交
999
    ekey = ascScan ? nextWin.ekey : nextWin.skey;
1000
    forwardRows =
1001
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
1002
    // window start(end) key interpolation
1003
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
L
Liu Jicong 已提交
1004
    // TODO: add to open window? how to close the open windows after input blocks exhausted?
S
shenglian zhou 已提交
1005
#if 0
1006 1007 1008 1009
    if ((ascScan && ekey <= pBlock->info.window.ekey) ||
        (!ascScan && ekey >= pBlock->info.window.skey)) {
      // window start(end) key interpolation
      doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
1010
    } else if (pInfo->timeWindowInterpo) {
1011 1012
      addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
    }
S
shenglian zhou 已提交
1013
#endif
1014
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
H
Haojun Liao 已提交
1015 1016
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
1017
    doCloseWindow(pResultRowInfo, pInfo, pResult);
1018 1019 1020
  }

  if (pInfo->timeWindowInterpo) {
1021
    saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
1022
  }
1023 1024 1025 1026 1027 1028 1029 1030 1031 1032
}

void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
  // current result is done in computing final results.
  if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
    closeResultRow(pResult);
    tdListPopHead(pResultRowInfo->openWindow);
  }
}

1033 1034 1035 1036 1037
SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, uint64_t groupId) {
  SOpenWindowInfo openWin = {0};
  openWin.pos.pageId = pResult->pageId;
  openWin.pos.offset = pResult->offset;
  openWin.groupId = groupId;
L
Liu Jicong 已提交
1038
  SListNode* pn = tdListGetTail(pResultRowInfo->openWindow);
1039
  if (pn == NULL) {
1040 1041
    tdListAppend(pResultRowInfo->openWindow, &openWin);
    return openWin.pos;
1042 1043
  }

L
Liu Jicong 已提交
1044
  SOpenWindowInfo* px = (SOpenWindowInfo*)pn->data;
1045 1046
  if (px->pos.pageId != openWin.pos.pageId || px->pos.offset != openWin.pos.offset || px->groupId != openWin.groupId) {
    tdListAppend(pResultRowInfo->openWindow, &openWin);
1047 1048
  }

1049
  return openWin.pos;
1050 1051 1052 1053
}

int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
  TSKEY* tsCols = NULL;
1054

1055 1056 1057 1058
  if (pBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;

1059 1060 1061 1062 1063 1064
    // no data in primary ts
    if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) {
      return NULL;
    }

    if (tsCols[0] != 0 && (pBlock->info.window.skey == 0 && pBlock->info.window.ekey == 0)) {
1065 1066 1067 1068 1069
      blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
    }
  }

  return tsCols;
1070 1071 1072 1073 1074 1075 1076
}

static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
  }

L
Liu Jicong 已提交
1077
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1078
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
1079
  SExprSupp*                pSup = &pOperator->exprSupp;
1080

1081 1082
  int32_t scanFlag = MAIN_SCAN;

X
Xiaoyu Wang 已提交
1083
  int64_t        st = taosGetTimestampUs();
1084 1085 1086
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1087
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1088 1089 1090 1091
    if (pBlock == NULL) {
      break;
    }

1092
    getTableScanInfo(pOperator, &pInfo->inputOrder, &scanFlag);
1093

1094
    if (pInfo->scalarSupp.pExprInfo != NULL) {
L
Liu Jicong 已提交
1095 1096
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
1097 1098
    }

1099
    // the pDataBlock are always the same one, no need to call this again
1100
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, scanFlag, true);
1101 1102
    blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);

1103
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag);
1104 1105
  }

1106
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->resultTsOrder);
1107
  OPTR_SET_OPENED(pOperator);
1108 1109

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1110 1111 1112
  return TSDB_CODE_SUCCESS;
}

1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
static bool compareVal(const char* v, const SStateKeys* pKey) {
  if (IS_VAR_DATA_TYPE(pKey->type)) {
    if (varDataLen(v) != varDataLen(pKey->pData)) {
      return false;
    } else {
      return strncmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0;
    }
  } else {
    return memcmp(pKey->pData, v, pKey->bytes) == 0;
  }
}

1125
static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
L
Liu Jicong 已提交
1126
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1127
  SExprSupp*     pSup = &pOperator->exprSupp;
1128

1129
  SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
1130 1131 1132
  int64_t          gid = pBlock->info.groupId;

  bool    masterScan = true;
1133
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1134 1135
  int16_t bytes = pStateColInfoData->info.bytes;

1136
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1137 1138 1139 1140 1141
  TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

1142
  struct SColumnDataAgg* pAgg = NULL;
1143
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
X
Xiaoyu Wang 已提交
1144
    pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
1145
    if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
1146 1147 1148 1149 1150
      continue;
    }

    char* val = colDataGetData(pStateColInfoData, j);

1151
    if (gid != pRowSup->groupId || !pInfo->hasKey) {
1152 1153 1154 1155 1156 1157 1158
      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }

1159 1160
      pInfo->hasKey = true;

1161 1162
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1163
    } else if (compareVal(val, &pInfo->stateKey)) {
1164
      doKeepTuple(pRowSup, tsList[j], gid);
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // a new state window started
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1175 1176
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1177
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1178
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1179 1180 1181
      }

      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1182 1183
      doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                       pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1184 1185

      // here we start a new session window
1186 1187
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1188 1189 1190 1191 1192 1193 1194

      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }
1195 1196 1197 1198 1199
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1200 1201
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1202
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1203
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1204 1205 1206
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
H
Haojun Liao 已提交
1207 1208
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
                   pBlock->info.rows, numOfOutput);
1209 1210
}

1211
static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
1212 1213 1214 1215 1216
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
1217

1218 1219
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  SExprSupp*     pSup = &pOperator->exprSupp;
1220

1221
  SOptrBasicInfo* pBInfo = &pInfo->binfo;
1222 1223

  if (pOperator->status == OP_RES_TO_RETURN) {
1224
    while (1) {
1225
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1226
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1227

1228
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }

      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1239
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1240 1241
  }

1242
  int32_t order = TSDB_ORDER_ASC;
1243
  int64_t st = taosGetTimestampUs();
1244 1245 1246

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  while (1) {
1247
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1248 1249 1250 1251
    if (pBlock == NULL) {
      break;
    }

1252
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1253 1254
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1255 1256 1257
    doStateWindowAggImpl(pOperator, pInfo, pBlock);
  }

X
Xiaoyu Wang 已提交
1258
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1259 1260
  pOperator->status = OP_RES_TO_RETURN;

1261
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1262
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1263
  while (1) {
1264
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1265
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1266

1267
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1268 1269 1270 1271
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
1272

1273 1274 1275 1276 1277
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1278
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1279 1280
}

1281
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
1282
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
1283
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1284 1285 1286 1287 1288 1289 1290 1291

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pBlock = pInfo->binfo.pRes;

  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
1292
    return pOperator->fpSet.getStreamResFn(pOperator);
1293 1294 1295 1296 1297 1298 1299
  } else {
    pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
    if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
      return NULL;
    }

    blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
1300 1301
    while (1) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1302
      doFilter(pInfo->pCondition, pBlock, NULL);
1303

1304
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1305 1306 1307 1308
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
1309

1310 1311 1312
      if (pBlock->info.rows > 0) {
        break;
      }
1313 1314
    }

1315 1316 1317
    size_t rows = pBlock->info.rows;
    pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1318
    return (rows == 0) ? NULL : pBlock;
1319 1320 1321 1322
  }
}

// todo merged with the build group result.
1323
static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
1324
                                  int32_t* rowEntryInfoOffset) {
1325 1326 1327 1328 1329 1330 1331
  size_t num = taosArrayGetSize(pUpdateList);

  for (int32_t i = 0; i < num; ++i) {
    SResKeyPos* pPos = taosArrayGetP(pUpdateList, i);

    SFilePage*  bufPage = getBufPage(pBuf, pPos->pos.pageId);
    SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
1332

1333
    for (int32_t j = 0; j < numOfOutput; ++j) {
1334
      SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, j, rowEntryInfoOffset);
1335 1336
      if (pRow->numOfRows < pEntry->numOfRes) {
        pRow->numOfRows = pEntry->numOfRes;
1337 1338 1339 1340 1341 1342
      }
    }

    releaseBufPage(pBuf, bufPage);
  }
}
5
54liuyao 已提交
1343
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
L
Liu Jicong 已提交
1344
  for (int i = 0; i < num; i++) {
5
54liuyao 已提交
1345 1346
    if (type == STREAM_INVERT) {
      fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
L
Liu Jicong 已提交
1347
    } else if (type == STREAM_NORMAL) {
5
54liuyao 已提交
1348 1349 1350 1351
      fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
    }
  }
}
5
54liuyao 已提交
1352

5
54liuyao 已提交
1353
static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) {
1354
  SResultRow*     pResult = getResultRowByPos(pResultBuf, p1, false);
1355
  SqlFunctionCtx* pCtx = pSup->pCtx;
5
54liuyao 已提交
1356
  for (int32_t i = 0; i < numOfOutput; ++i) {
1357
    pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
1358 1359 1360 1361 1362 1363 1364 1365 1366
    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
      continue;
    }
    pResInfo->initialized = false;
    if (pCtx[i].functionId != -1) {
      pCtx[i].fpSet.init(&pCtx[i], pResInfo);
    }
  }
5
54liuyao 已提交
1367 1368 1369
  SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pResultBuf, bufPage);
5
54liuyao 已提交
1370 1371
}

5
54liuyao 已提交
1372 1373
static bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
                          int32_t numOfOutput) {
1374
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId);
5
54liuyao 已提交
1375
  SResultRowPosition* p1 =
1376
      (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1377 1378
  if (!p1) {
    // window has been closed
5
54liuyao 已提交
1379
    return false;
1380
  }
1381
  doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput);
5
54liuyao 已提交
1382
  return true;
5
54liuyao 已提交
1383 1384
}

5
54liuyao 已提交
1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419
static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, int32_t numOfOutput) {
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
  SWinKey                      key = {.ts = ts, .groupId = groupId};
  tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));
  streamStateDel(pOperator->pTaskInfo->streamInfo.pState, &key);
  return true;
}

static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int32_t numOfOutput, SSDataBlock* pBlock,
                            SArray* pUpWins, SHashObj* pUpdatedMap) {
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  TSKEY*           startTsCols = (TSKEY*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           endTsCols = (TSKEY*)pEndTsCol->pData;
  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*        pGpDatas = (uint64_t*)pGpCol->pData;
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
    while (win.ekey <= endTsCols[i]) {
      uint64_t winGpId = pGpDatas[i];
      bool     res = doDeleteWindow(pOperator, win.skey, winGpId, numOfOutput);
      SWinKey  winRes = {.ts = win.skey, .groupId = winGpId};
      if (pUpWins && res) {
        taosArrayPush(pUpWins, &winRes);
      }
      if (pUpdatedMap) {
        taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
    }
  }
}

1420 1421 1422 1423
bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) {
  size_t bytes = sizeof(TSKEY);
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId);
  SResultRowPosition* p1 =
1424
      (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1425 1426 1427 1428
  if (!p1) {
    // window has been closed
    return false;
  }
1429
  tSimpleHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1430 1431 1432
  return true;
}

5
54liuyao 已提交
1433 1434
static void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, STimeWindowAggSupp* pTwSup, SSDataBlock* pBlock,
                                          SArray* pDelWins, SInterval* pInterval, SHashObj* pUpdatedMap) {
1435
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
1436
  TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
1437 1438
  SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           tsEnds = (TSKEY*)pEndCol->pData;
1439
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
1440
  uint64_t*        groupIds = (uint64_t*)pGroupCol->pData;
5
54liuyao 已提交
1441
  int64_t          numOfWin = tSimpleHashGetSize(pAggSup->pResultRowHashTable);
1442
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
5
54liuyao 已提交
1443 1444
    TSKEY          startTs = TMAX(tsStarts[i], pTwSup->minTs);
    TSKEY          endTs = TMIN(tsEnds[i], pTwSup->maxTs);
1445 1446
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
5
54liuyao 已提交
1447
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTs, pInterval, TSDB_ORDER_ASC);
1448 1449 1450 1451 1452 1453 1454 1455 1456 1457
    do {
      doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
      SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]};
      if (pDelWins) {
        taosArrayPush(pDelWins, &winRes);
      }
      if (pUpdatedMap) {
        taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
5
54liuyao 已提交
1458
    } while (win.skey <= endTs);
1459 1460 1461
  }
}

1462 1463
static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput,
                           SSDataBlock* pBlock, SArray* pUpWins) {
1464 1465 1466 1467
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  TSKEY*           startTsCols = (TSKEY*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           endTsCols = (TSKEY*)pEndTsCol->pData;
1468 1469
  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*        pGpDatas = (uint64_t*)pGpCol->pData;
1470 1471 1472 1473 1474
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
    while (win.ekey <= endTsCols[i]) {
1475
      uint64_t winGpId = pGpDatas[i];
L
Liu Jicong 已提交
1476
      bool     res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput);
1477
      if (pUpWins && res) {
H
Haojun Liao 已提交
1478
        SWinKey winRes = {.ts = win.skey, .groupId = winGpId};
1479 1480 1481
        taosArrayPush(pUpWins, &winRes);
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
5
54liuyao 已提交
1482
    }
5
54liuyao 已提交
1483 1484
  }
}
1485

1486 1487 1488 1489 1490 1491
static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) {
  void*   pIte = NULL;
  size_t  keyLen = 0;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    void*    key = tSimpleHashGetKey(pIte, &keyLen);
5
54liuyao 已提交
1492 1493
    uint64_t groupId = *(uint64_t*)key;
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
1494
    TSKEY               ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1495
    SResultRowPosition* pPos = (SResultRowPosition*)pIte;
5
54liuyao 已提交
1496
    int32_t             code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
5
54liuyao 已提交
1497 1498 1499 1500 1501 1502 1503
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }
  return TSDB_CODE_SUCCESS;
}

1504
static int32_t closeIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
5
54liuyao 已提交
1505
                                   SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pRecyPages,
1506
                                   SDiskbasedBuf* pDiscBuf) {
1507
  qDebug("===stream===close interval window");
1508 1509 1510 1511 1512
  void*   pIte = NULL;
  size_t  keyLen = 0;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    void*    key = tSimpleHashGetKey(pIte, &keyLen);
X
Xiaoyu Wang 已提交
1513
    uint64_t groupId = *(uint64_t*)key;
5
54liuyao 已提交
1514
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
L
Liu Jicong 已提交
1515
    TSKEY       ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1516 1517 1518
    STimeWindow win;
    win.skey = ts;
    win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
H
Haojun Liao 已提交
1519
    SWinKey winRe = {
L
Liu Jicong 已提交
1520 1521
        .ts = win.skey,
        .groupId = groupId,
L
Liu Jicong 已提交
1522
    };
H
Haojun Liao 已提交
1523
    void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinKey));
5
54liuyao 已提交
1524
    if (isCloseWindow(&win, pSup)) {
5
54liuyao 已提交
1525
      if (chIds && pPullDataMap) {
L
Liu Jicong 已提交
1526
        SArray* chAy = *(SArray**)chIds;
5
54liuyao 已提交
1527
        int32_t size = taosArrayGetSize(chAy);
5
54liuyao 已提交
1528
        qDebug("===stream===window %" PRId64 " wait child size:%d", win.skey, size);
5
54liuyao 已提交
1529
        for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
1530
          qDebug("===stream===window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i));
5
54liuyao 已提交
1531 1532 1533
        }
        continue;
      } else if (pPullDataMap) {
5
54liuyao 已提交
1534
        qDebug("===stream===close window %" PRId64, win.skey);
5
54liuyao 已提交
1535
      }
5
54liuyao 已提交
1536 1537
      SResultRowPosition* pPos = (SResultRowPosition*)pIte;
      if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
1538
        int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, closeWins);
5
54liuyao 已提交
1539 1540 1541
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
1542 1543 1544
        ASSERT(pRecyPages != NULL);
        taosArrayPush(pRecyPages, &pPos->pageId);
      } else {
5
54liuyao 已提交
1545
        // SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId);
1546
        // dBufSetBufPageRecycled(pDiscBuf, bufPage);
5
54liuyao 已提交
1547
      }
5
54liuyao 已提交
1548 1549
      char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
      SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
1550
      tSimpleHashIterateRemove(pHashMap, keyBuf, keyLen, &pIte, &iter);
5
54liuyao 已提交
1551 1552 1553 1554 1555
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598
static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval,
                                         SHashObj* pPullDataMap, SHashObj* closeWins, SOperatorInfo* pOperator) {
  qDebug("===stream===close interval window");
  void*   pIte = NULL;
  size_t  keyLen = 0;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    void*       key = tSimpleHashGetKey(pIte, &keyLen);
    SWinKey*    pWinKey = (SWinKey*)key;
    void*       chIds = taosHashGet(pPullDataMap, pWinKey, sizeof(SWinKey));
    STimeWindow win = {
        .skey = pWinKey->ts,
        .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1,
    };
    if (isCloseWindow(&win, pTwSup)) {
      if (chIds && pPullDataMap) {
        SArray* chAy = *(SArray**)chIds;
        int32_t size = taosArrayGetSize(chAy);
        qDebug("===stream===window %" PRId64 " wait child size:%d", pWinKey->ts, size);
        for (int32_t i = 0; i < size; i++) {
          qDebug("===stream===window %" PRId64 " wait child id:%d", pWinKey->ts, *(int32_t*)taosArrayGet(chAy, i));
        }
        continue;
      } else if (pPullDataMap) {
        qDebug("===stream===close window %" PRId64, pWinKey->ts);
      }

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
        int32_t code = saveWinResultInfo(pWinKey->ts, pWinKey->groupId, closeWins);
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
      }
      tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter);

      if (needDeleteWindowBuf(&win, pTwSup)) {
        streamStateDel(pOperator->pTaskInfo->streamInfo.pState, pWinKey);
      }
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
1599 1600 1601
static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) {
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
L
Liu Jicong 已提交
1602
    SOperatorInfo*                    pChildOp = taosArrayGetP(pChildren, i);
5
54liuyao 已提交
1603 1604 1605
    SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
    ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
1606 1607
    closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, NULL,
                        pChInfo->aggSup.pResultBuf);
1608 1609 1610 1611 1612 1613 1614
  }
}

static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) {
  int32_t size = taosArrayGetSize(pageIds);
  for (int32_t i = 0; i < size; i++) {
    int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i);
5
54liuyao 已提交
1615
    // SFilePage* bufPage = getBufPage(pDiskBuf, pageId);
1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629
    // dBufSetBufPageRecycled(pDiskBuf, bufPage);
  }
  taosArrayClear(pageIds);
}

static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlock) {
  blockDataCleanup(pBlock);
  int32_t size = taosArrayGetSize(pWins);
  if (*index == size) {
    *index = 0;
    taosArrayClear(pWins);
    return;
  }
  blockDataEnsureCapacity(pBlock, size - *index);
1630
  uint64_t uid = 0;
1631
  for (int32_t i = *index; i < size; i++) {
H
Haojun Liao 已提交
1632
    SWinKey* pWin = taosArrayGet(pWins, i);
1633
    appendOneRow(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId);
1634
    (*index)++;
5
54liuyao 已提交
1635 1636 1637
  }
}

1638
static void destroyStateWindowOperatorInfo(void* param) {
1639
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
1640
  cleanupBasicInfo(&pInfo->binfo);
1641
  taosMemoryFreeClear(pInfo->stateKey.pData);
1642

D
dapan1121 已提交
1643
  taosMemoryFreeClear(param);
1644 1645
}

H
Haojun Liao 已提交
1646
static void freeItem(void* param) {
L
Liu Jicong 已提交
1647
  SGroupKeys* pKey = (SGroupKeys*)param;
H
Haojun Liao 已提交
1648 1649 1650
  taosMemoryFree(pKey->pData);
}

1651
void destroyIntervalOperatorInfo(void* param) {
1652
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
1653
  cleanupBasicInfo(&pInfo->binfo);
1654
  cleanupAggSup(&pInfo->aggSup);
1655 1656 1657 1658
  cleanupExprSupp(&pInfo->scalarSupp);

  tdListFree(pInfo->binfo.resultRowInfo.openWindow);

H
Haojun Liao 已提交
1659 1660 1661 1662 1663 1664 1665
  pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages);
  pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols);
  taosArrayDestroyEx(pInfo->pPrevValues, freeItem);

  pInfo->pPrevValues = NULL;
  pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins);
  pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);
1666

H
Haojun Liao 已提交
1667 1668
  cleanupGroupResInfo(&pInfo->groupResInfo);
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
D
dapan1121 已提交
1669
  taosMemoryFreeClear(param);
1670 1671
}

1672
void destroyStreamFinalIntervalOperatorInfo(void* param) {
X
Xiaoyu Wang 已提交
1673
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
1674
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
1675
  cleanupAggSup(&pInfo->aggSup);
L
Liu Jicong 已提交
1676
  // it should be empty.
5
54liuyao 已提交
1677 1678 1679
  taosHashCleanup(pInfo->pPullDataMap);
  taosArrayDestroy(pInfo->pPullWins);
  blockDataDestroy(pInfo->pPullDataRes);
1680
  taosArrayDestroy(pInfo->pRecycledPages);
H
Haojun Liao 已提交
1681
  blockDataDestroy(pInfo->pUpdateRes);
L
Liu Jicong 已提交
1682 1683
  taosArrayDestroy(pInfo->pDelWins);
  blockDataDestroy(pInfo->pDelRes);
5
54liuyao 已提交
1684

1685 1686 1687 1688
  if (pInfo->pChildren) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
1689
      destroyStreamFinalIntervalOperatorInfo(pChildOp->info);
L
Liu Jicong 已提交
1690 1691
      taosMemoryFree(pChildOp->pDownstream);
      cleanupExprSupp(&pChildOp->exprSupp);
1692 1693
      taosMemoryFreeClear(pChildOp);
    }
L
Liu Jicong 已提交
1694
    taosArrayDestroy(pInfo->pChildren);
1695
  }
1696
  nodesDestroyNode((SNode*)pInfo->pPhyNode);
5
54liuyao 已提交
1697
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
1698

D
dapan1121 已提交
1699
  taosMemoryFreeClear(param);
5
54liuyao 已提交
1700 1701
}

1702
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
5
54liuyao 已提交
1703
  for (int32_t i = 0; i < numOfCols; i++) {
5
54liuyao 已提交
1704
    if (fmIsUserDefinedFunc(pFCtx[i].functionId) || !fmIsInvertible(pFCtx[i].functionId)) {
5
54liuyao 已提交
1705 1706 1707 1708 1709 1710
      return false;
    }
  }
  return true;
}

1711
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
1712 1713
  // the primary timestamp column
  bool needed = false;
1714 1715
  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
1716

X
Xiaoyu Wang 已提交
1717
  {  // ts column
1718 1719
    SColumn c = {0};
    c.colId = 1;
1720
    c.slotId = pInfo->primaryTsIndex;
1721 1722
    c.type = TSDB_DATA_TYPE_TIMESTAMP;
    c.bytes = sizeof(int64_t);
1723
    taosArrayPush(pInfo->pInterpCols, &c);
1724 1725

    SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1726 1727 1728 1729
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = true;  // to denote no value is assigned yet
    key.pData = taosMemoryCalloc(1, c.bytes);
1730
    taosArrayPush(pInfo->pPrevValues, &key);
1731 1732
  }

X
Xiaoyu Wang 已提交
1733
  for (int32_t i = 0; i < numOfCols; ++i) {
1734 1735
    SExprInfo* pExpr = pCtx[i].pExpr;

H
Haojun Liao 已提交
1736
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1737 1738 1739
      SFunctParam* pParam = &pExpr->base.pParam[0];

      SColumn c = *pParam->pCol;
1740
      taosArrayPush(pInfo->pInterpCols, &c);
1741 1742 1743
      needed = true;

      SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1744 1745
      key.bytes = c.bytes;
      key.type = c.type;
1746
      key.isNull = false;
X
Xiaoyu Wang 已提交
1747
      key.pData = taosMemoryCalloc(1, c.bytes);
1748
      taosArrayPush(pInfo->pPrevValues, &key);
1749 1750 1751 1752 1753 1754
    }
  }

  return needed;
}

L
Liu Jicong 已提交
1755
void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval,
5
54liuyao 已提交
1756
                            STimeWindowAggSupp* pTwSup) {
1757
  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
5
54liuyao 已提交
1758
    initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, pTwSup);
1759 1760
    return;
  }
5
54liuyao 已提交
1761
  SStreamScanInfo* pScanInfo = downstream->info;
1762 1763
  pScanInfo->windowSup.parentType = type;
  pScanInfo->windowSup.pIntervalAggSup = pSup;
5
54liuyao 已提交
1764
  pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, pTwSup->waterMark);
1765
  pScanInfo->interval = *pInterval;
5
54liuyao 已提交
1766
  pScanInfo->twAggSup = *pTwSup;
5
54liuyao 已提交
1767 1768
}

H
Haojun Liao 已提交
1769 1770 1771 1772 1773 1774
void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
  for (int32_t i = 0; i < numOfExpr; i++) {
    pCtx[i].isStream = true;
  }
}

1775 1776
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                          SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
L
Liu Jicong 已提交
1777 1778
                                          STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode,
                                          SExecTaskInfo* pTaskInfo, bool isStream) {
1779
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1780
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1781 1782 1783 1784
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1785
  pOperator->pTaskInfo = pTaskInfo;
L
Liu Jicong 已提交
1786
  pInfo->win = pTaskInfo->window;
1787 1788
  pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
  pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
L
Liu Jicong 已提交
1789
  pInfo->interval = *pInterval;
L
Liu Jicong 已提交
1790
  pInfo->execModel = pTaskInfo->execModel;
L
Liu Jicong 已提交
1791
  pInfo->twAggSup = *pTwAggSupp;
5
54liuyao 已提交
1792
  pInfo->ignoreExpiredData = pPhyNode->window.igExpired;
1793
  pInfo->pCondition = pPhyNode->window.node.pConditions;
1794
  pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock;
1795 1796 1797 1798

  if (pPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
L
Liu Jicong 已提交
1799
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
1800 1801 1802 1803
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
1804

1805
  pInfo->primaryTsIndex = primaryTsSlotId;
1806 1807
  SExprSupp* pSup = &pOperator->exprSupp;

1808
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
1809
  initResultSizeInfo(&pOperator->resultInfo, 4096);
1810

1811
  int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
1812 1813 1814 1815
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

1816 1817
  initBasicInfo(&pInfo->binfo, pResBlock);

1818 1819
  if (isStream) {
    ASSERT(numOfCols > 0);
H
Haojun Liao 已提交
1820
    initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
1821
  }
1822

1823
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
1824

1825
  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
X
Xiaoyu Wang 已提交
1826
  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
1827

1828
  pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, pInfo);
1829
  if (pInfo->timeWindowInterpo) {
1830
    pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
H
Haojun Liao 已提交
1831 1832 1833
    if (pInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
1834
  }
1835

1836
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
H
Haojun Liao 已提交
1837
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
1838
  pInfo->delIndex = 0;
1839
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
1840
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1841

X
Xiaoyu Wang 已提交
1842 1843 1844 1845 1846
  pOperator->name = "TimeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
1847

1848
  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL,
1849 1850 1851 1852 1853 1854 1855 1856 1857
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1858
_error:
1859
  destroyIntervalOperatorInfo(pInfo);
1860 1861 1862 1863 1864
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

1865
// todo handle multiple timeline cases. assume no timeline interweaving
1866 1867
static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1868
  SExprSupp*     pSup = &pOperator->exprSupp;
1869

1870
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1871 1872

  bool    masterScan = true;
1873
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888
  int64_t gid = pBlock->info.groupId;

  int64_t gap = pInfo->gap;

  if (!pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->winSup.prevTs = INT64_MIN;
  }

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

  // In case of ascending or descending order scan data, only one time window needs to be kepted for each table.
  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
1889 1890 1891
    if (gid != pRowSup->groupId || pInfo->winSup.prevTs == INT64_MIN) {
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
H
Haojun Liao 已提交
1892 1893
    } else if (((tsList[j] - pRowSup->prevTs >= 0) && (tsList[j] - pRowSup->prevTs <= gap)) ||
               ((pRowSup->prevTs - tsList[j] >= 0) && (pRowSup->prevTs - tsList[j] <= gap))) {
1894
      // The gap is less than the threshold, so it belongs to current session window that has been opened already.
1895
      doKeepTuple(pRowSup, tsList[j], gid);
1896 1897 1898 1899 1900 1901 1902 1903 1904 1905
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1906 1907
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1908
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1909
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1910 1911 1912 1913
      }

      // pInfo->numOfRows data belong to the current session window
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1914 1915
      doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                       pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1916 1917

      // here we start a new session window
1918 1919
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1920 1921 1922 1923 1924
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1925 1926
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1927
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1928
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1929 1930 1931
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
H
Haojun Liao 已提交
1932 1933
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
                   pBlock->info.rows, numOfOutput);
1934 1935
}

1936
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
1937 1938 1939 1940 1941 1942
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*          pBInfo = &pInfo->binfo;
1943
  SExprSupp*               pSup = &pOperator->exprSupp;
1944 1945

  if (pOperator->status == OP_RES_TO_RETURN) {
1946
    while (1) {
1947
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1948
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1949

1950
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1951 1952 1953 1954
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
1955

1956 1957 1958 1959 1960
      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1961
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1962 1963
  }

1964 1965 1966
  int64_t st = taosGetTimestampUs();
  int32_t order = TSDB_ORDER_ASC;

1967 1968 1969
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1970
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1971 1972 1973 1974 1975
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1976
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1977 1978
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1979 1980 1981
    doSessionWindowAggImpl(pOperator, pInfo, pBlock);
  }

1982 1983
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;

1984 1985 1986
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;

1987
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1988
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1989
  while (1) {
1990
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1991
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1992

1993
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1994 1995 1996 1997
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
1998

1999 2000 2001 2002 2003
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
2004
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
2005 2006
}

2007
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
H
Haojun Liao 已提交
2008
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2009 2010
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
H
Haojun Liao 已提交
2011 2012 2013 2014 2015 2016

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);

      pkey->isNull = false;
2017
      char* val = colDataGetData(pColInfoData, rowIndex);
H
Haojun Liao 已提交
2018 2019 2020
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }
2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040

  pSliceInfo->isPrevRowSet = true;
}

static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i);

      pkey->isNull = false;
      char* val = colDataGetData(pColInfoData, rowIndex);
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }

  pSliceInfo->isNextRowSet = true;
H
Haojun Liao 已提交
2041 2042
}

2043 2044
static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex,
                             bool isLastRow) {
2045
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
dengyihao's avatar
dengyihao 已提交
2046
  bool    fillLastPoint = pSliceInfo->fillLastPoint;
2047 2048
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
2049 2050
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
    SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i);
2051 2052 2053

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069
      if (isLastRow) {
        pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
      } else if (fillLastPoint) {
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
      } else {
        pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1);

        char* val;
        val = colDataGetData(pColInfoData, rowIndex);
        memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes);
        val = colDataGetData(pColInfoData, rowIndex + 1);
        memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes);
      }
2070 2071 2072 2073

      pLinearInfo->hasNull = false;
    } else {
      pLinearInfo->hasNull = true;
2074 2075 2076
    }
  }

2077
  pSliceInfo->fillLastPoint = isLastRow ? true : false;
2078 2079
}

dengyihao's avatar
dengyihao 已提交
2080
static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) {
2081
  int32_t rows = pResBlock->info.rows;
2082
  blockDataEnsureCapacity(pResBlock, rows + 1);
2083 2084 2085
  // todo set the correct primary timestamp column

  // output the result
2086
  bool hasInterp = true;
2087 2088 2089
  for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
    SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
    int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
2090
    int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
2091

dengyihao's avatar
dengyihao 已提交
2092
    // SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
2093 2094 2095
    SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

    switch (pSliceInfo->fillType) {
2096
      case TSDB_FILL_NULL: {
2097 2098
        colDataAppendNULL(pDst, rows);
        break;
2099
      }
2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116

      case TSDB_FILL_SET_VALUE: {
        SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal;

        if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
          float v = 0;
          GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
          double v = 0;
          GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
          int64_t v = 0;
          GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        }
2117 2118
        break;
      }
2119

2120
      case TSDB_FILL_LINEAR: {
2121
        SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, srcSlot);
2122

dengyihao's avatar
dengyihao 已提交
2123 2124
        SPoint start = pLinearInfo->start;
        SPoint end = pLinearInfo->end;
2125 2126 2127
        SPoint current = {.key = pSliceInfo->current};
        current.val = taosMemoryCalloc(pLinearInfo->bytes, 1);

2128 2129
        // before interp range, do not fill
        if (start.key == INT64_MIN || end.key == INT64_MAX) {
2130
          hasInterp = false;
2131 2132 2133
          break;
        }

2134 2135 2136 2137
        if (pLinearInfo->hasNull) {
          colDataAppendNULL(pDst, rows);
        } else {
          taosGetLinearInterpolationVal(&current, pLinearInfo->type, &start, &end, pLinearInfo->type);
dengyihao's avatar
dengyihao 已提交
2138
          colDataAppend(pDst, rows, (char*)current.val, false);
2139 2140
        }

2141
        taosMemoryFree(current.val);
2142
        break;
2143
      }
2144
      case TSDB_FILL_PREV: {
2145
        if (!pSliceInfo->isPrevRowSet) {
2146
          hasInterp = false;
2147 2148 2149
          break;
        }

2150 2151
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
        colDataAppend(pDst, rows, pkey->pData, false);
2152 2153
        break;
      }
2154 2155

      case TSDB_FILL_NEXT: {
2156
        if (!pSliceInfo->isNextRowSet) {
2157
          hasInterp = false;
2158 2159 2160
          break;
        }

2161 2162
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot);
        colDataAppend(pDst, rows, pkey->pData, false);
2163 2164
        break;
      }
2165 2166 2167 2168 2169 2170

      case TSDB_FILL_NONE:
      default:
        break;
    }
  }
2171 2172 2173 2174

  if (hasInterp) {
    pResBlock->info.rows += 1;
  }
2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186
}

static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pPrevRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pPrevRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

2187
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pPrevRow, &key);
  }

2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227
  pInfo->isPrevRowSet = false;

  return TSDB_CODE_SUCCESS;
}

static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pNextRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pNextRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pNextRow, &key);
  }

  pInfo->isNextRowSet = false;

2228 2229 2230
  return TSDB_CODE_SUCCESS;
}

2231 2232 2233 2234 2235 2236
static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pLinearInfo != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo));
2237
  if (pInfo->pLinearInfo == NULL) {
2238 2239 2240 2241 2242 2243 2244
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

2245 2246
    SFillLinearInfo linearInfo = {0};
    linearInfo.start.key = INT64_MIN;
dengyihao's avatar
dengyihao 已提交
2247
    linearInfo.end.key = INT64_MAX;
2248
    linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes);
dengyihao's avatar
dengyihao 已提交
2249 2250 2251
    linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes);
    linearInfo.hasNull = false;
    linearInfo.type = pColInfo->info.type;
2252 2253
    linearInfo.bytes = pColInfo->info.bytes;
    taosArrayPush(pInfo->pLinearInfo, &linearInfo);
2254 2255
  }

2256 2257
  pInfo->fillLastPoint = false;

2258 2259 2260
  return TSDB_CODE_SUCCESS;
}

2261 2262 2263 2264
static bool needToFillLastPoint(STimeSliceOperatorInfo* pSliceInfo) {
  return (pSliceInfo->fillLastPoint == true && pSliceInfo->fillType == TSDB_FILL_LINEAR);
}

2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281
static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  int32_t code;
  code = initPrevRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initNextRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initFillLinearInfo(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

2282 2283 2284
  return TSDB_CODE_SUCCESS;
}

H
Haojun Liao 已提交
2285
static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
2286 2287 2288 2289
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

2290 2291
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

2292
  STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
2293 2294
  SSDataBlock*            pResBlock = pSliceInfo->pRes;
  SExprSupp*              pSup = &pOperator->exprSupp;
H
Haojun Liao 已提交
2295

2296 2297
  //  if (pOperator->status == OP_RES_TO_RETURN) {
  //    //    doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes);
2298
  //    if (pResBlock->info.rows == 0 || !hasRemainResults(&pSliceInfo->groupResInfo)) {
2299 2300 2301 2302 2303
  //      doSetOperatorCompleted(pOperator);
  //    }
  //
  //    return pResBlock;
  //  }
2304

2305 2306
  int32_t        order = TSDB_ORDER_ASC;
  SInterval*     pInterval = &pSliceInfo->interval;
2307 2308
  SOperatorInfo* downstream = pOperator->pDownstream[0];

2309 2310
  blockDataCleanup(pResBlock);

2311
  while (1) {
2312
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
2313 2314 2315 2316
    if (pBlock == NULL) {
      break;
    }

2317
    int32_t code = initKeeperInfo(pSliceInfo, pBlock);
2318
    if (code != TSDB_CODE_SUCCESS) {
2319
      T_LONG_JMP(pTaskInfo->env, code);
2320 2321
    }

2322
    // the pDataBlock are always the same one, no need to call this again
2323
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
H
Haojun Liao 已提交
2324

2325
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
2326 2327
    for (int32_t i = 0; i < pBlock->info.rows; ++i) {
      int64_t ts = *(int64_t*)colDataGetData(pTsCol, i);
H
Haojun Liao 已提交
2328

dengyihao's avatar
dengyihao 已提交
2329
      if (i == 0 && needToFillLastPoint(pSliceInfo)) {  // first row in current block
2330 2331 2332 2333 2334 2335 2336 2337 2338
        doKeepLinearInfo(pSliceInfo, pBlock, i, false);
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
          genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
        }
2339
      }
2340

2341 2342 2343
      if (pSliceInfo->current > pSliceInfo->win.ekey) {
        doSetOperatorCompleted(pOperator);
        break;
2344 2345
      }

H
Haojun Liao 已提交
2346
      if (ts == pSliceInfo->current) {
2347
        for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
2348
          SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
2349 2350
          int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
          int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
H
Haojun Liao 已提交
2351 2352

          SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
2353
          SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
H
Haojun Liao 已提交
2354

2355 2356 2357 2358 2359
          if (colDataIsNull_s(pSrc, i)) {
            colDataAppendNULL(pDst, pResBlock->info.rows);
            continue;
          }

H
Haojun Liao 已提交
2360
          char* v = colDataGetData(pSrc, i);
2361
          colDataAppend(pDst, pResBlock->info.rows, v, false);
H
Haojun Liao 已提交
2362 2363
        }

2364
        pResBlock->info.rows += 1;
2365
        doKeepPrevRows(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2366

2367 2368
        // for linear interpolation, always fill value between this and next points;
        // if its the first point in data block, also fill values between previous(if there's any) and this point;
2369 2370
        // if its the last point in data block, no need to fill, but reserve this point as the start value and do
        // the interpolation when processing next data block.
2371
        if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2372 2373
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2374
          if (i < pBlock->info.rows - 1) {
2375
            doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2376 2377 2378
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2379
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2380 2381
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2382 2383 2384 2385
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }
2386

2387 2388 2389 2390 2391
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
            }
dengyihao's avatar
dengyihao 已提交
2392 2393
          } else {  // it is the last row of current block
            // store ts value as start, and calculate interp value when processing next block
2394
            doKeepLinearInfo(pSliceInfo, pBlock, i, true);
2395
          }
dengyihao's avatar
dengyihao 已提交
2396
        } else {  // non-linear interpolation
2397 2398 2399 2400 2401 2402
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (pSliceInfo->current > pSliceInfo->win.ekey) {
            doSetOperatorCompleted(pOperator);
            break;
          }
2403

2404 2405 2406
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
2407
        }
H
Haojun Liao 已提交
2408
      } else if (ts < pSliceInfo->current) {
2409
        // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate
2410 2411
        doKeepPrevRows(pSliceInfo, pBlock, i);

2412
        if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2413
          // no need to increate pSliceInfo->current here
dengyihao's avatar
dengyihao 已提交
2414
          // pSliceInfo->current =
2415 2416
          //    taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (i < pBlock->info.rows - 1) {
2417
            doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2418 2419 2420
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2421
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2422 2423
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2424 2425 2426 2427 2428 2429 2430
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }

              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
H
Haojun Liao 已提交
2431 2432
                break;
              }
H
Haojun Liao 已提交
2433
            }
2434
          }
dengyihao's avatar
dengyihao 已提交
2435
        } else {  // non-linear interpolation
2436 2437 2438 2439 2440 2441
          if (i < pBlock->info.rows - 1) {
            // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
            doKeepNextRows(pSliceInfo, pBlock, i + 1);
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2442
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2443 2444
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2445 2446 2447 2448
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }
2449

2450 2451 2452 2453 2454 2455
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
            } else {
              // ignore current row, and do nothing
H
Haojun Liao 已提交
2456
            }
2457 2458
          } else {  // it is the last row of current block
            doKeepPrevRows(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2459
          }
2460 2461
        }
      } else {  // ts > pSliceInfo->current
2462
        // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
2463 2464
        doKeepNextRows(pSliceInfo, pBlock, i);

2465
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
2466
          genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
2467 2468
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2469 2470 2471
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
2472 2473
        }

2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490
        // add current row if timestamp match
        if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
          for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
            SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
            int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
            int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;

            SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
            SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

            char* v = colDataGetData(pSrc, i);
            colDataAppend(pDst, pResBlock->info.rows, v, false);
          }

          pResBlock->info.rows += 1;
          doKeepPrevRows(pSliceInfo, pBlock, i);

2491 2492 2493 2494
          if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
            pSliceInfo->current =
                taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
            if (i < pBlock->info.rows - 1) {
2495
              doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2496 2497 2498
              int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
              if (nextTs > pSliceInfo->current) {
                while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2499
                  genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2500 2501
                  pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                    pInterval->precision);
2502 2503 2504 2505
                  if (pResBlock->info.rows >= pResBlock->info.capacity) {
                    break;
                  }
                }
2506

2507 2508 2509 2510 2511 2512
                if (pSliceInfo->current > pSliceInfo->win.ekey) {
                  doSetOperatorCompleted(pOperator);
                  break;
                }
              }
            }
dengyihao's avatar
dengyihao 已提交
2513
          } else {  // non-linear interpolation
2514 2515 2516 2517 2518 2519
            pSliceInfo->current =
                taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);

            if (pResBlock->info.rows >= pResBlock->info.capacity) {
              break;
            }
2520 2521 2522
          }
        }

2523 2524 2525
        if (pSliceInfo->current > pSliceInfo->win.ekey) {
          doSetOperatorCompleted(pOperator);
          break;
H
Haojun Liao 已提交
2526 2527 2528
        }
      }
    }
2529
  }
2530

2531 2532
  // check if need to interpolate after last datablock
  // except for fill(next), fill(linear)
dengyihao's avatar
dengyihao 已提交
2533 2534
  while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT &&
         pSliceInfo->fillType != TSDB_FILL_LINEAR) {
2535 2536 2537 2538 2539
    genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
    pSliceInfo->current =
        taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
    if (pResBlock->info.rows >= pResBlock->info.capacity) {
      break;
2540
    }
2541 2542 2543 2544
  }

  // restore the value
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
2545
  if (pResBlock->info.rows == 0) {
2546 2547 2548
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
2549 2550 2551
  return pResBlock->info.rows == 0 ? NULL : pResBlock;
}

2552
void destroyTimeSliceOperatorInfo(void* param) {
2553 2554 2555 2556 2557 2558 2559 2560
  STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param;

  pInfo->pRes = blockDataDestroy(pInfo->pRes);

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) {
    SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i);
    taosMemoryFree(pKey->pData);
  }
2561
  taosArrayDestroy(pInfo->pPrevRow);
2562 2563 2564 2565 2566 2567

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pNextRow); ++i) {
    SGroupKeys* pKey = taosArrayGet(pInfo->pNextRow, i);
    taosMemoryFree(pKey->pData);
  }
  taosArrayDestroy(pInfo->pNextRow);
2568 2569 2570 2571 2572 2573

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pLinearInfo); ++i) {
    SFillLinearInfo* pKey = taosArrayGet(pInfo->pLinearInfo, i);
    taosMemoryFree(pKey->start.val);
    taosMemoryFree(pKey->end.val);
  }
2574 2575
  taosArrayDestroy(pInfo->pLinearInfo);

2576
  taosMemoryFree(pInfo->pFillColInfo);
2577 2578 2579
  taosMemoryFreeClear(param);
}

2580
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
2581 2582 2583 2584 2585 2586
  STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pOperator == NULL || pInfo == NULL) {
    goto _error;
  }

2587
  SInterpFuncPhysiNode* pInterpPhyNode = (SInterpFuncPhysiNode*)pPhyNode;
2588
  SExprSupp*            pSup = &pOperator->exprSupp;
2589

2590
  int32_t    numOfExprs = 0;
2591
  SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
2592
  int32_t    code = initExprSupp(pSup, pExprInfo, numOfExprs);
H
Haojun Liao 已提交
2593 2594 2595 2596
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2597
  if (pInterpPhyNode->pExprs != NULL) {
2598
    int32_t    num = 0;
2599 2600 2601 2602 2603 2604 2605 2606 2607
    SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

  pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries);
  pInfo->fillType = convertFillType(pInterpPhyNode->fillMode);
2608
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2609

H
Haojun Liao 已提交
2610
  pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, NULL, 0, (SNodeListNode*)pInterpPhyNode->pFillValues);
2611
  pInfo->pLinearInfo = NULL;
L
Liu Jicong 已提交
2612 2613
  pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  pInfo->win = pInterpPhyNode->timeRange;
2614
  pInfo->interval.interval = pInterpPhyNode->interval;
L
Liu Jicong 已提交
2615
  pInfo->current = pInfo->win.skey;
H
Haojun Liao 已提交
2616

2617
  pOperator->name = "TimeSliceOperator";
2618
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC;
2619 2620 2621 2622
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
  pOperator->pTaskInfo = pTaskInfo;
2623

2624
  pOperator->fpSet =
2625
      createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyTimeSliceOperatorInfo, NULL, NULL, NULL);
2626

2627 2628
  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);

H
Haojun Liao 已提交
2629
  code = appendDownstream(pOperator, &downstream, 1);
2630 2631
  return pOperator;

L
Liu Jicong 已提交
2632
_error:
2633 2634 2635 2636 2637 2638
  taosMemoryFree(pInfo);
  taosMemoryFree(pOperator);
  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
  return NULL;
}

2639 2640
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
                                             SExecTaskInfo* pTaskInfo) {
2641 2642 2643 2644 2645 2646
  SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

L
Liu Jicong 已提交
2647
  int32_t      num = 0;
2648 2649 2650 2651 2652 2653 2654
  SExprInfo*   pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc);
  int32_t      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;

  SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
2655 2656 2657
  pInfo->stateKey.type = pInfo->stateCol.type;
  pInfo->stateKey.bytes = pInfo->stateCol.bytes;
  pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
2658
  pInfo->pCondition = pStateNode->window.node.pConditions;
2659 2660 2661 2662
  if (pInfo->stateKey.pData == NULL) {
    goto _error;
  }

2663 2664
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

2665
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2666
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
2667 2668 2669 2670
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2671
  initBasicInfo(&pInfo->binfo, pResBlock);
2672
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2673

L
Liu Jicong 已提交
2674 2675 2676
  pInfo->twAggSup =
      (STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType};
  ;
2677 2678
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

X
Xiaoyu Wang 已提交
2679 2680
  pInfo->tsSlotId = tsSlotId;
  pOperator->name = "StateWindowOperator";
2681
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE;
X
Xiaoyu Wang 已提交
2682 2683 2684 2685
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
2686 2687 2688 2689

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL,
                                         destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

2690 2691 2692 2693 2694
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2695 2696
  return pOperator;

L
Liu Jicong 已提交
2697
_error:
2698 2699 2700
  destroyStateWindowOperatorInfo(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
2701 2702 2703
  return NULL;
}

2704
void destroySWindowOperatorInfo(void* param) {
2705
  SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
2706 2707 2708
  if (pInfo == NULL) {
    return;
  }
2709

2710
  cleanupBasicInfo(&pInfo->binfo);
H
Haojun Liao 已提交
2711 2712 2713 2714
  colDataDestroy(&pInfo->twAggSup.timeWindowData);

  cleanupAggSup(&pInfo->aggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
D
dapan1121 已提交
2715
  taosMemoryFreeClear(param);
2716 2717
}

H
Haojun Liao 已提交
2718
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode,
2719
                                            SExecTaskInfo* pTaskInfo) {
2720 2721 2722 2723 2724 2725
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

2726
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
2727
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2728

2729
  int32_t      numOfCols = 0;
H
Haojun Liao 已提交
2730 2731 2732
  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc);

2733
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
2734 2735 2736 2737
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2738 2739
  initBasicInfo(&pInfo->binfo, pResBlock);

H
Haojun Liao 已提交
2740 2741 2742 2743
  pInfo->twAggSup.waterMark = pSessionNode->window.watermark;
  pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType;
  pInfo->gap = pSessionNode->gap;

2744
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2745 2746
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

2747
  pInfo->tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
L
Liu Jicong 已提交
2748 2749 2750
  pInfo->binfo.pRes = pResBlock;
  pInfo->winSup.prevTs = INT64_MIN;
  pInfo->reptScan = false;
2751
  pInfo->pCondition = pSessionNode->window.node.pConditions;
H
Haojun Liao 已提交
2752

L
Liu Jicong 已提交
2753
  pOperator->name = "SessionWindowAggOperator";
2754
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
2755
  pOperator->blocking = true;
L
Liu Jicong 已提交
2756 2757
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
2758 2759 2760 2761 2762

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL,
                                         destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  pOperator->pTaskInfo = pTaskInfo;
  code = appendDownstream(pOperator, &downstream, 1);
2763 2764 2765 2766
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2767 2768
  return pOperator;

L
Liu Jicong 已提交
2769
_error:
2770
  destroySWindowOperatorInfo(pInfo);
2771 2772 2773
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
2774
}
5
54liuyao 已提交
2775

5
54liuyao 已提交
2776
void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
2777
                      SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) {
5
54liuyao 已提交
2778 2779
  for (int32_t k = 0; k < numOfOutput; ++k) {
    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
2780 2781 2782 2783 2784
      if (!pTimeWindowData) {
        continue;
      }

      SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);
L
Liu Jicong 已提交
2785 2786
      char*                p = GET_ROWCELL_INTERBUF(pEntryInfo);
      SColumnInfoData      idata = {0};
2787 2788 2789 2790 2791 2792 2793 2794
      idata.info.type = TSDB_DATA_TYPE_BIGINT;
      idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
      idata.pData = p;

      SScalarParam out = {.columnData = &idata};
      SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
      pDestCtx[k].sfp.process(&tw, 1, &out);
      pEntryInfo->numOfRes = 1;
L
Liu Jicong 已提交
2795
    } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
2796
      int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
5
54liuyao 已提交
2797 2798 2799
      if (code != TSDB_CODE_SUCCESS) {
        qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
        pTaskInfo->code = code;
2800
        T_LONG_JMP(pTaskInfo->env, code);
5
54liuyao 已提交
2801 2802 2803 2804 2805
      }
    }
  }
}

2806 2807 2808 2809
bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) {
  int32_t bytes = sizeof(TSKEY);
  SET_RES_WINDOW_KEY(pSup->keyBuf, &ts, bytes, groupId);
  SResultRowPosition* p1 =
2810
      (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
2811 2812 2813
  return p1 != NULL;
}

2814 2815 2816 2817 2818 2819
STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
  return w;
}

2820
static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray,
L
Liu Jicong 已提交
2821 2822
                                  int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo,
                                  SHashObj* pUpdatedMap) {
5
54liuyao 已提交
2823
  int32_t size = taosArrayGetSize(pWinArray);
5
54liuyao 已提交
2824 2825 2826
  if (!pInfo->pChildren) {
    return;
  }
5
54liuyao 已提交
2827
  for (int32_t i = 0; i < size; i++) {
H
Haojun Liao 已提交
2828
    SWinKey*    pWinRes = taosArrayGet(pWinArray, i);
2829
    SResultRow* pCurResult = NULL;
2830 2831 2832 2833 2834
    STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval);
    if (isDeletedWindow(&parentWin, pWinRes->groupId, &pInfo->aggSup) && isCloseWindow(&parentWin, &pInfo->twAggSup)) {
      continue;
    }
    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &parentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx,
2835
                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2836
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
2837
    int32_t num = 0;
5
54liuyao 已提交
2838 2839 2840
    for (int32_t j = 0; j < numOfChildren; j++) {
      SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
2841
      SExprSupp*                pChildSup = &pChildOp->exprSupp;
2842 2843 2844
      if (!hasIntervalWindow(&pChInfo->aggSup, pWinRes->ts, pWinRes->groupId)) {
        continue;
      }
2845
      num++;
2846
      SResultRow* pChResult = NULL;
2847
      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &parentWin, true, &pChResult, pWinRes->groupId,
2848 2849
                             pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup,
                             pTaskInfo);
2850 2851
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true);
      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
2852
    }
2853
    if (num > 0 && pUpdatedMap) {
2854
      saveWinResultRow(pCurResult, pWinRes->groupId, pUpdatedMap);
D
dapan1121 已提交
2855
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur);
2856
    }
5
54liuyao 已提交
2857 2858 2859 2860 2861
  }
}

bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
  SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId);
2862
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
L
Liu Jicong 已提交
2863
                                                               GET_RES_WINDOW_KEY_LEN(sizeof(int64_t)));
5
54liuyao 已提交
2864 2865 2866
  return p1 == NULL;
}

5
54liuyao 已提交
2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SOperatorInfo* pOperator, STimeWindowAggSupp* pTwSup) {
  if (pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {
    SWinKey key = {.ts = pWin->skey, .groupId = groupId};
    void*   pVal = NULL;
    int32_t size = 0;
    if (streamStateGet(pOperator->pTaskInfo->streamInfo.pState, &key, &pVal, &size) < 0) {
      return false;
    }
    streamStateReleaseBuf(pOperator->pTaskInfo->streamInfo.pState, &key, pVal);
  }
  return false;
}

L
Liu Jicong 已提交
2880 2881 2882 2883
int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,
                        STimeWindow* pNextWin) {
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2884 2885 2886 2887
  int32_t prevEndPos = forwardRows - 1 + startPos;
  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
}

H
Haojun Liao 已提交
2888
void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {
5
54liuyao 已提交
2889 2890 2891 2892
  SArray* childIds = taosArrayInit(8, sizeof(int32_t));
  for (int32_t i = 0; i < size; i++) {
    taosArrayPush(childIds, &i);
  }
H
Haojun Liao 已提交
2893
  taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*));
5
54liuyao 已提交
2894 2895 2896 2897
}

static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

2898 2899
static void doHashIntervalAgg(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
                              SHashObj* pUpdatedMap) {
5
54liuyao 已提交
2900
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
X
Xiaoyu Wang 已提交
2901 2902
  SResultRowInfo*                   pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*                    pTaskInfo = pOperatorInfo->pTaskInfo;
2903 2904
  SExprSupp*                        pSup = &pOperatorInfo->exprSupp;
  int32_t                           numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
2905 2906 2907 2908
  int32_t                           step = 1;
  TSKEY*                            tsCols = NULL;
  SResultRow*                       pResult = NULL;
  int32_t                           forwardRows = 0;
5
54liuyao 已提交
2909

5
54liuyao 已提交
2910 2911 2912
  ASSERT(pSDataBlock->pDataBlock != NULL);
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
2913

2914
  int32_t     startPos = 0;
X
Xiaoyu Wang 已提交
2915
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
5
54liuyao 已提交
2916 2917 2918 2919 2920 2921
  STimeWindow nextWin = {0};
  if (IS_FINAL_OP(pInfo)) {
    nextWin = getFinalTimeWindow(ts, &pInfo->interval);
  } else {
    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order);
  }
5
54liuyao 已提交
2922
  while (1) {
5
54liuyao 已提交
2923
    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
5
54liuyao 已提交
2924
    if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
5
54liuyao 已提交
2925 2926 2927 2928 2929 2930 2931
      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
      if (startPos < 0) {
        break;
      }
      continue;
    }
    if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) {
L
Liu Jicong 已提交
2932
      bool    ignore = true;
H
Haojun Liao 已提交
2933
      SWinKey winRes = {
L
Liu Jicong 已提交
2934 2935 2936
          .ts = nextWin.skey,
          .groupId = tableGroupId,
      };
H
Haojun Liao 已提交
2937
      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
2938 2939 2940
      if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId};
        // add pull data request
5
54liuyao 已提交
2941
        savePullWindow(&pull, pInfo->pPullWins);
5
54liuyao 已提交
2942 2943 2944
        int32_t size = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, &winRes, size);
        qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size);
5
54liuyao 已提交
2945 2946 2947
      } else {
        int32_t index = -1;
        SArray* chArray = NULL;
5
54liuyao 已提交
2948
        int32_t chId = 0;
5
54liuyao 已提交
2949
        if (chIds) {
L
Liu Jicong 已提交
2950
          chArray = *(void**)chIds;
5
54liuyao 已提交
2951
          chId = getChildIndex(pSDataBlock);
5
54liuyao 已提交
2952 2953
          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
        }
L
Liu Jicong 已提交
2954
        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {
5
54liuyao 已提交
2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965
          ignore = false;
        }
      }

      if (ignore) {
        startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
        if (startPos < 0) {
          break;
        }
        continue;
      }
5
54liuyao 已提交
2966
    }
5
54liuyao 已提交
2967

2968 2969
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx,
                                          numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2970
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
2971
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
2972
    }
5
54liuyao 已提交
2973

5
54liuyao 已提交
2974 2975 2976
    if (IS_FINAL_OP(pInfo)) {
      forwardRows = 1;
    } else {
H
Hongze Cheng 已提交
2977 2978
      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,
                                             NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2979
    }
5
54liuyao 已提交
2980 2981
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
D
dapan1121 已提交
2982
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
5
54liuyao 已提交
2983
    }
5
54liuyao 已提交
2984
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
2985 2986
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pSDataBlock->info.rows, numOfOutput);
2987
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
2988
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
5
54liuyao 已提交
2989 2990 2991 2992 2993 2994 2995
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
    if (startPos < 0) {
      break;
    }
  }
}

5
54liuyao 已提交
2996
static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) {
2997
  tSimpleHashClear(pInfo->aggSup.pResultRowHashTable);
5
54liuyao 已提交
2998
  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
2999
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
L
Liu Jicong 已提交
3000
  pInfo->aggSup.currentPageId = -1;
5
54liuyao 已提交
3001 3002
}

5
54liuyao 已提交
3003 3004 3005 3006
static void clearSpecialDataBlock(SSDataBlock* pBlock) {
  if (pBlock->info.rows <= 0) {
    return;
  }
5
54liuyao 已提交
3007 3008 3009
  blockDataCleanup(pBlock);
}

3010
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
5
54liuyao 已提交
3011 3012
  // ASSERT(pDest->info.capacity >= pSource->info.rows);
  blockDataEnsureCapacity(pDest, pSource->info.rows);
5
54liuyao 已提交
3013
  clearSpecialDataBlock(pDest);
5
54liuyao 已提交
3014 3015
  SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
  SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
3016

5
54liuyao 已提交
3017
  // copy timestamp column
3018 3019
  colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info);
  for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) {
5
54liuyao 已提交
3020 3021 3022
    SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
    colDataAppendNNULL(pCol, 0, pSource->info.rows);
  }
3023

5
54liuyao 已提交
3024
  pDest->info.rows = pSource->info.rows;
3025 3026
  pDest->info.groupId = pSource->info.groupId;
  pDest->info.type = pSource->info.type;
5
54liuyao 已提交
3027 3028 3029
  blockDataUpdateTsWindow(pDest, 0);
}

5
54liuyao 已提交
3030 3031 3032 3033 3034 3035
static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
  clearSpecialDataBlock(pBlock);
  int32_t size = taosArrayGetSize(array);
  if (size - (*pIndex) == 0) {
    return;
  }
L
Liu Jicong 已提交
3036
  blockDataEnsureCapacity(pBlock, size - (*pIndex));
5
54liuyao 已提交
3037
  ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
3038 3039 3040 3041 3042
  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
5
54liuyao 已提交
3043
  for (; (*pIndex) < size; (*pIndex)++) {
L
Liu Jicong 已提交
3044
    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));
5
54liuyao 已提交
3045 3046 3047
    colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
    colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
3048 3049
    colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
5
54liuyao 已提交
3050 3051 3052 3053 3054 3055 3056 3057 3058
    pBlock->info.rows++;
  }
  if ((*pIndex) == size) {
    *pIndex = 0;
    taosArrayClear(array);
  }
  blockDataUpdateTsWindow(pBlock, 0);
}

L
Liu Jicong 已提交
3059
void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
5
54liuyao 已提交
3060
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
L
Liu Jicong 已提交
3061
  TSKEY*           tsData = (TSKEY*)pStartCol->pData;
5
54liuyao 已提交
3062
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
L
Liu Jicong 已提交
3063 3064
  uint64_t*        groupIdData = (uint64_t*)pGroupCol->pData;
  int32_t          chId = getChildIndex(pBlock);
5
54liuyao 已提交
3065
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
H
Haojun Liao 已提交
3066 3067
    SWinKey winRes = {.ts = tsData[i], .groupId = groupIdData[i]};
    void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
3068
    if (chIds) {
L
Liu Jicong 已提交
3069
      SArray* chArray = *(SArray**)chIds;
5
54liuyao 已提交
3070 3071
      int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
      if (index != -1) {
5
54liuyao 已提交
3072
        qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId);
5
54liuyao 已提交
3073 3074 3075
        taosArrayRemove(chArray, index);
        if (taosArrayGetSize(chArray) == 0) {
          // pull data is over
H
Haojun Liao 已提交
3076
          taosHashRemove(pMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
3077 3078 3079 3080 3081
        }
      }
    }
  }
}
5
54liuyao 已提交
3082

3083 3084 3085
static void addRetriveWindow(SArray* wins, SStreamFinalIntervalOperatorInfo* pInfo) {
  int32_t size = taosArrayGetSize(wins);
  for (int32_t i = 0; i < size; i++) {
L
Liu Jicong 已提交
3086
    SWinKey*    winKey = taosArrayGet(wins, i);
3087 3088 3089 3090 3091 3092 3093
    STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
    if (isCloseWindow(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) {
      void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
      if (!chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId};
        // add pull data request
        savePullWindow(&pull, pInfo->pPullWins);
3094 3095 3096
        int32_t size1 = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, winKey, size1);
        qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, size1);
3097 3098 3099 3100 3101
      }
    }
  }
}

5
54liuyao 已提交
3102 3103 3104 3105 3106 3107
static void clearFunctionContext(SExprSupp* pSup) {
  for (int32_t i = 0; i < pSup->numOfExprs; i++) {
    pSup->pCtx[i].saveHandle.currentPage = -1;
  }
}

5
54liuyao 已提交
3108 3109
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
  SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
3110 3111 3112 3113 3114 3115

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  SArray*        pUpdated = taosArrayInit(4, POINTER_BYTES);
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
  TSKEY          maxTs = INT64_MIN;
5
54liuyao 已提交
3116
  TSKEY          minTs = INT64_MAX;
5
54liuyao 已提交
3117

3118 3119
  SExprSupp* pSup = &pOperator->exprSupp;

5
54liuyao 已提交
3120
  qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
L
Liu Jicong 已提交
3121

5
54liuyao 已提交
3122 3123 3124
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
3125 3126 3127 3128
    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
    if (pInfo->pPullDataRes->info.rows != 0) {
      // process the rest of the data
      ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3129
      printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3130 3131 3132
      return pInfo->pPullDataRes;
    }

3133 3134 3135 3136 3137 3138 3139
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
      return pInfo->pDelRes;
    }

5
54liuyao 已提交
3140
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3141
    if (pInfo->binfo.pRes->info.rows == 0) {
5
54liuyao 已提交
3142
      pOperator->status = OP_EXEC_DONE;
5
54liuyao 已提交
3143
      if (!IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
3144
        clearFunctionContext(&pOperator->exprSupp);
5
54liuyao 已提交
3145 3146
        // semi interval operator clear disk buffer
        clearStreamIntervalOperator(pInfo);
3147
        qDebug("===stream===clear semi operator");
3148 3149
      } else {
        freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3150
      }
5
54liuyao 已提交
3151 3152
      return NULL;
    }
5
54liuyao 已提交
3153
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3154 3155
    return pInfo->binfo.pRes;
  } else {
5
54liuyao 已提交
3156 3157 3158
    if (!IS_FINAL_OP(pInfo)) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
      if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3159
        printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3160 3161
        return pInfo->binfo.pRes;
      }
5
54liuyao 已提交
3162 3163 3164 3165
    }
    if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
      pInfo->returnUpdate = false;
      ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3166
      printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3167 3168
      // process the rest of the data
      return pInfo->pUpdateRes;
5
54liuyao 已提交
3169
    }
3170 3171 3172
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
5
54liuyao 已提交
3173
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
3174 3175
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
3176 3177 3178 3179 3180
  }

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
3181
      clearSpecialDataBlock(pInfo->pUpdateRes);
5
54liuyao 已提交
3182
      removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
5
54liuyao 已提交
3183
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
3184
      qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3185 3186
      break;
    }
5
54liuyao 已提交
3187
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv");
3188

H
Haojun Liao 已提交
3189 3190
    ASSERT(pBlock->info.type != STREAM_INVERT);
    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
5
54liuyao 已提交
3191 3192
      pInfo->binfo.pRes->info.type = pBlock->info.type;
    } else if (pBlock->info.type == STREAM_CLEAR) {
H
Haojun Liao 已提交
3193
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinKey));
3194
      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
5
54liuyao 已提交
3195
      if (IS_FINAL_OP(pInfo)) {
L
Liu Jicong 已提交
3196 3197
        int32_t                           childIndex = getChildIndex(pBlock);
        SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
3198
        SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
L
Liu Jicong 已提交
3199
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;
3200

3201 3202 3203
        doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildSup->numOfExprs, pBlock, NULL);
        rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs,
                              pOperator->pTaskInfo, NULL);
5
54liuyao 已提交
3204 3205
        taosArrayDestroy(pUpWins);
        continue;
3206
      }
5
54liuyao 已提交
3207
      removeResults(pUpWins, pUpdatedMap);
3208
      copyDataBlock(pInfo->pUpdateRes, pBlock);
5
54liuyao 已提交
3209
      pInfo->returnUpdate = true;
3210
      taosArrayDestroy(pUpWins);
5
54liuyao 已提交
3211
      break;
3212
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
3213
      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
5
54liuyao 已提交
3214
      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, delWins, &pInfo->interval, pUpdatedMap);
3215 3216 3217 3218 3219
      if (IS_FINAL_OP(pInfo)) {
        int32_t                           childIndex = getChildIndex(pBlock);
        SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;
5
54liuyao 已提交
3220
        doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, &pInfo->twAggSup, pBlock, NULL, &pChildInfo->interval, NULL);
L
Liu Jicong 已提交
3221 3222
        rebuildIntervalWindow(pInfo, pSup, delWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs,
                              pOperator->pTaskInfo, pUpdatedMap);
3223 3224 3225
        addRetriveWindow(delWins, pInfo);
        taosArrayAddAll(pInfo->pDelWins, delWins);
        taosArrayDestroy(delWins);
3226 3227
        continue;
      }
3228 3229 3230
      removeResults(delWins, pUpdatedMap);
      taosArrayAddAll(pInfo->pDelWins, delWins);
      taosArrayDestroy(delWins);
3231
      break;
5
54liuyao 已提交
3232
    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
3233
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
5
54liuyao 已提交
3234
      continue;
5
54liuyao 已提交
3235
    } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
H
Haojun Liao 已提交
3236
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinKey));
5
54liuyao 已提交
3237
      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
5
54liuyao 已提交
3238
      removeResults(pUpWins, pUpdatedMap);
5
54liuyao 已提交
3239 3240 3241 3242 3243
      taosArrayDestroy(pUpWins);
      if (taosArrayGetSize(pUpdated) > 0) {
        break;
      }
      continue;
L
Liu Jicong 已提交
3244 3245
    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) {
      processPullOver(pBlock, pInfo->pPullDataMap);
5
54liuyao 已提交
3246
      continue;
5
54liuyao 已提交
3247
    }
5
54liuyao 已提交
3248

5
54liuyao 已提交
3249 3250 3251 3252
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
3253
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
3254
    doHashIntervalAgg(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
5
54liuyao 已提交
3255
    if (IS_FINAL_OP(pInfo)) {
S
shenglian zhou 已提交
3256
      int32_t chIndex = getChildIndex(pBlock);
5
54liuyao 已提交
3257 3258 3259 3260 3261
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
        SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
        if (!pChildOp) {
3262
          T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3263
        }
3264 3265
        SStreamFinalIntervalOperatorInfo* pTmpInfo = pChildOp->info;
        pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3266
        taosArrayPush(pInfo->pChildren, &pChildOp);
5
54liuyao 已提交
3267
        qDebug("===stream===add child, id:%d", chIndex);
5
54liuyao 已提交
3268
      }
S
shenglian zhou 已提交
3269
      SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
3270
      SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
3271
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
3272
      doHashIntervalAgg(pChildOp, pBlock, pBlock->info.groupId, NULL);
5
54liuyao 已提交
3273
    }
5
54liuyao 已提交
3274 3275 3276
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
    maxTs = TMAX(maxTs, pBlock->info.watermark);
    minTs = TMIN(minTs, pBlock->info.window.skey);
5
54liuyao 已提交
3277
  }
S
shenglian zhou 已提交
3278

5
54liuyao 已提交
3279
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
3280
  pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs);
5
54liuyao 已提交
3281
  if (IS_FINAL_OP(pInfo)) {
3282
    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap,
5
54liuyao 已提交
3283
                        pUpdatedMap, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3284
    closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
3285 3286
  } else {
    pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;
5
54liuyao 已提交
3287 3288
  }

5
54liuyao 已提交
3289 3290 3291 3292 3293 3294 3295
  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosHashCleanup(pUpdatedMap);
  taosArraySort(pUpdated, resultrowComparAsc);

3296
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
3297 3298
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
3299 3300 3301 3302 3303

  doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
  if (pInfo->pPullDataRes->info.rows != 0) {
    // process the rest of the data
    ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3304
    printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3305 3306 3307
    return pInfo->pPullDataRes;
  }

3308
  // we should send result first.
5
54liuyao 已提交
3309
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3310
  if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3311
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3312 3313 3314 3315 3316 3317
    return pInfo->binfo.pRes;
  }

  if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
    pInfo->returnUpdate = false;
    ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3318
    printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3319 3320 3321
    // process the rest of the data
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
3322

3323 3324 3325
  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows != 0) {
    // process the rest of the data
5
54liuyao 已提交
3326
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
3327 3328
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
3329 3330 3331
  return NULL;
}

S
shenglian zhou 已提交
3332 3333 3334
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                     SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
  SIntervalPhysiNode*               pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
5
54liuyao 已提交
3335
  SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo));
S
shenglian zhou 已提交
3336
  SOperatorInfo*                    pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3337 3338 3339
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
3340

3341
  pOperator->pTaskInfo = pTaskInfo;
5
54liuyao 已提交
3342
  pInfo->order = TSDB_ORDER_ASC;
S
shenglian zhou 已提交
3343 3344 3345 3346 3347 3348 3349 3350
  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                .sliding = pIntervalPhyNode->sliding,
                                .intervalUnit = pIntervalPhyNode->intervalUnit,
                                .slidingUnit = pIntervalPhyNode->slidingUnit,
                                .offset = pIntervalPhyNode->offset,
                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pIntervalPhyNode->window.watermark,
5
54liuyao 已提交
3351 3352
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
3353
      .minTs = INT64_MAX,
S
shenglian zhou 已提交
3354
  };
3355
  ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY);
5
54liuyao 已提交
3356 3357
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
3358
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3359 3360 3361 3362 3363 3364 3365 3366 3367
  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

S
shenglian zhou 已提交
3368 3369
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5
54liuyao 已提交
3370
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
3371 3372

  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
3373 3374 3375 3376
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
3377
  initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs);
3378 3379
  initBasicInfo(&pInfo->binfo, pResBlock);

3380
  ASSERT(numOfCols > 0);
5
54liuyao 已提交
3381
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
3382

3383
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3384 3385
  pInfo->pChildren = NULL;
  if (numOfChild > 0) {
3386
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
5
54liuyao 已提交
3387 3388 3389
    for (int32_t i = 0; i < numOfChild; i++) {
      SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp) {
3390 3391
        SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
        pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3392 3393 3394 3395 3396 3397
        taosArrayPush(pInfo->pChildren, &pChildOp);
        continue;
      }
      goto _error;
    }
  }
3398
  pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
3399
  blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
5
54liuyao 已提交
3400 3401
  pInfo->returnUpdate = false;

3402
  pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
5
54liuyao 已提交
3403

5
54liuyao 已提交
3404 3405 3406 3407
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
    pInfo->isFinal = true;
    pOperator->name = "StreamFinalIntervalOperator";
  } else {
5
54liuyao 已提交
3408
    // semi interval operator does not catch result
5
54liuyao 已提交
3409 3410
    pInfo->isFinal = false;
    pOperator->name = "StreamSemiIntervalOperator";
H
Haojun Liao 已提交
3411
    ASSERT(pInfo->aggSup.currentPageId == -1);
5
54liuyao 已提交
3412 3413
  }

5
54liuyao 已提交
3414
  if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
5
54liuyao 已提交
3415 3416
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }
5
54liuyao 已提交
3417 3418 3419 3420
  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
  pInfo->pullIndex = 0;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
3421
  pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
5
54liuyao 已提交
3422
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
3423
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3424
  pInfo->delIndex = 0;
H
Haojun Liao 已提交
3425
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
5
54liuyao 已提交
3426
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
5
54liuyao 已提交
3427

5
54liuyao 已提交
3428
  pOperator->operatorType = pPhyNode->type;
5
54liuyao 已提交
3429 3430 3431 3432
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;

S
shenglian zhou 已提交
3433 3434 3435
  pOperator->fpSet =
      createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
3436
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
5
54liuyao 已提交
3437
    initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
3438
  }
5
54liuyao 已提交
3439 3440 3441 3442 3443 3444 3445 3446
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
3447
  destroyStreamFinalIntervalOperatorInfo(pInfo);
5
54liuyao 已提交
3448 3449 3450
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
5
54liuyao 已提交
3451
}
5
54liuyao 已提交
3452 3453 3454

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
3455
  void** pIte = NULL;
3456
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
3457
    SArray* pWins = (SArray*)(*pIte);
3458 3459 3460
    taosArrayDestroy(pWins);
  }
  taosHashCleanup(pSup->pResultRows);
5
54liuyao 已提交
3461
  destroyDiskbasedBuf(pSup->pResultBuf);
3462
  blockDataDestroy(pSup->pScanBlock);
5
54liuyao 已提交
3463 3464
}

5
54liuyao 已提交
3465 3466 3467 3468
void destroyStateWinInfo(void* ptr) {
  if (ptr == NULL) {
    return;
  }
3469
  SStateWindowInfo* pWin = (SStateWindowInfo*)ptr;
5
54liuyao 已提交
3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484
  taosMemoryFreeClear(pWin->stateKey.pData);
}

void destroyStateStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
  void** pIte = NULL;
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
    SArray* pWins = (SArray*)(*pIte);
    taosArrayDestroyEx(pWins, (FDelete)destroyStateWinInfo);
  }
  taosHashCleanup(pSup->pResultRows);
  destroyDiskbasedBuf(pSup->pResultBuf);
  blockDataDestroy(pSup->pScanBlock);
}

3485
void destroyStreamSessionAggOperatorInfo(void* param) {
5
54liuyao 已提交
3486
  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;
3487
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
3488 3489
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
3490 3491 3492
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
3493
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
3494
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
3495
      destroyStreamSessionAggOperatorInfo(pChInfo);
3496 3497 3498
      taosMemoryFreeClear(pChild);
    }
  }
5
54liuyao 已提交
3499 3500 3501 3502
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  blockDataDestroy(pInfo->pWinBlock);
  blockDataDestroy(pInfo->pUpdateRes);
5
54liuyao 已提交
3503 3504
  destroySqlFunctionCtx(pInfo->pDummyCtx, 0);
  taosHashCleanup(pInfo->pStDeleted);
3505

D
dapan1121 已提交
3506
  taosMemoryFreeClear(param);
5
54liuyao 已提交
3507 3508
}

3509 3510
int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,
                        SSDataBlock* pResultBlock) {
3511 3512 3513 3514
  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
  if (code != TSDB_CODE_SUCCESS) {
    return code;
  }
H
Haojun Liao 已提交
3515
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
3516

3517
  initBasicInfo(pBasicInfo, pResultBlock);
3518

5
54liuyao 已提交
3519
  for (int32_t i = 0; i < numOfCols; ++i) {
3520
    pSup->pCtx[i].saveHandle.pBuf = NULL;
5
54liuyao 已提交
3521
  }
3522

3523
  ASSERT(numOfCols > 0);
5
54liuyao 已提交
3524 3525 3526 3527 3528 3529 3530 3531
  return TSDB_CODE_SUCCESS;
}

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {
  for (int i = 0; i < nums; i++) {
    pDummy[i].functionId = pCtx[i].functionId;
  }
}
5
54liuyao 已提交
3532

X
Xiaoyu Wang 已提交
3533
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
3534 3535 3536 3537 3538 3539 3540 3541 3542 3543
                    uint16_t type, int32_t tsColIndex) {
  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
    pScanInfo->tsColIndex = tsColIndex;
  }

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
    initDownStream(downstream->pDownstream[0], pAggSup, gap, waterMark, type, tsColIndex);
    return;
  }
3544
  SStreamScanInfo* pScanInfo = downstream->info;
3545
  pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
5
54liuyao 已提交
3546
  pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
5
54liuyao 已提交
3547 3548
}

3549 3550
int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx,
                                int32_t numOfOutput) {
3551 3552 3553
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo));
}

3554 3555 3556 3557 3558 3559 3560
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                  SExecTaskInfo* pTaskInfo) {
  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
  int32_t                        numOfCols = 0;
  SExprInfo*                     pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock*                   pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;
X
Xiaoyu Wang 已提交
3561
  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
3562
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3563 3564 3565 3566
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

3567 3568
  pOperator->pTaskInfo = pTaskInfo;

3569
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3570 3571 3572
  if (pSessionNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar);
3573
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
5
54liuyao 已提交
3574 3575 3576 3577
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
3578
  SExprSupp* pSup = &pOperator->exprSupp;
5
54liuyao 已提交
3579

3580
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
3581 3582 3583
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
3584

3585
  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
3586 3587 3588
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
X
Xiaoyu Wang 已提交
3589

5
54liuyao 已提交
3590 3591 3592 3593
  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }
3594
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
3595

3596
  pInfo->twAggSup = (STimeWindowAggSupp){
5
54liuyao 已提交
3597 3598 3599 3600 3601
      .waterMark = pSessionNode->window.watermark,
      .calTrigger = pSessionNode->window.triggerType,
      .maxTs = INT64_MIN,
      .minTs = INT64_MAX,
  };
H
Haojun Liao 已提交
3602 3603

  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3604 3605
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

5
54liuyao 已提交
3606 3607 3608 3609
  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  if (pSessionNode->window.pTsEnd) {
    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;
  }
3610
  pInfo->gap = pSessionNode->gap;
5
54liuyao 已提交
3611 3612 3613 3614 3615
  pInfo->binfo.pRes = pResBlock;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
5
54liuyao 已提交
3616
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3617
  pInfo->pChildren = NULL;
5
54liuyao 已提交
3618 3619
  pInfo->isFinal = false;
  pInfo->pPhyNode = pPhyNode;
5
54liuyao 已提交
3620
  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
5
54liuyao 已提交
3621 3622

  pOperator->name = "StreamSessionWindowAggOperator";
3623
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
5
54liuyao 已提交
3624 3625 3626
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
3627 3628 3629
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
3630
  if (downstream) {
L
Liu Jicong 已提交
3631 3632
    initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType,
                   pInfo->primaryTsIndex);
5
54liuyao 已提交
3633 3634
    code = appendDownstream(pOperator, &downstream, 1);
  }
5
54liuyao 已提交
3635 3636 3637 3638
  return pOperator;

_error:
  if (pInfo != NULL) {
3639
    destroyStreamSessionAggOperatorInfo(pInfo);
5
54liuyao 已提交
3640 3641 3642 3643 3644 3645 3646 3647
  }

  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

int64_t getSessionWindowEndkey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
3648
  SArray*            pWinInfos = (SArray*)data;
5
54liuyao 已提交
3649 3650 3651
  SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index);
  return pWin->win.ekey;
}
5
54liuyao 已提交
3652 3653

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {
5
54liuyao 已提交
3654
  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {
5
54liuyao 已提交
3655 3656 3657 3658 3659
    return true;
  }
  return false;
}

3660
bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); }
5
54liuyao 已提交
3661

3662
static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs, int32_t index) {
L
Liu Jicong 已提交
3663 3664
  SResultWindowInfo win = {
      .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
5
54liuyao 已提交
3665 3666 3667
  return taosArrayInsert(pWinInfos, index, &win);
}

3668
static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs) {
L
Liu Jicong 已提交
3669 3670
  SResultWindowInfo win = {
      .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
5
54liuyao 已提交
3671 3672 3673
  return taosArrayPush(pWinInfos, &win);
}

3674
SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) {
3675
  void**  ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t));
3676 3677 3678
  SArray* pWinInfos = NULL;
  if (ite == NULL) {
    pWinInfos = taosArrayInit(1024, pAggSup->valueSize);
3679
    taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void*));
3680 3681 3682 3683 3684 3685
  } else {
    pWinInfos = *ite;
  }
  return pWinInfos;
}

5
54liuyao 已提交
3686 3687
// don't add new window
SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
3688
                                       int64_t gap, int32_t* pIndex) {
3689 3690
  STimeWindow searchWin = {.skey = startTs, .ekey = endTs};
  SArray*     pWinInfos = getWinInfos(pAggSup, groupId);
5
54liuyao 已提交
3691 3692 3693 3694 3695 3696 3697
  pAggSup->pCurWins = pWinInfos;

  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    return NULL;
  }
  // find the first position which is smaller than the key
3698
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3699 3700 3701
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
3702
    if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) {
5
54liuyao 已提交
3703 3704 3705 3706 3707 3708 3709
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
3710
    if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) {
5
54liuyao 已提交
3711 3712 3713
      *pIndex = index + 1;
      return pWin;
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
3714
      *pIndex = index + 1;
5
54liuyao 已提交
3715 3716 3717 3718 3719 3720 3721
      return pWin;
    }
  }

  return NULL;
}

3722 3723
SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
                                        int64_t gap, int32_t* pIndex) {
3724 3725 3726
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

5
54liuyao 已提交
3727 3728
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
5
54liuyao 已提交
3729
    *pIndex = 0;
3730
    return addNewSessionWindow(pWinInfos, startTs, endTs);
5
54liuyao 已提交
3731 3732
  }
  // find the first position which is smaller than the key
5
54liuyao 已提交
3733
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3734 3735 3736
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
5
54liuyao 已提交
3737
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3738 3739 3740 3741 3742 3743 3744
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
5
54liuyao 已提交
3745
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3746 3747
      *pIndex = index + 1;
      return pWin;
5
54liuyao 已提交
3748 3749 3750
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
      *pIndex = index;
      return pWin;
5
54liuyao 已提交
3751 3752 3753 3754 3755
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
3756
    return addNewSessionWindow(pWinInfos, startTs, endTs);
5
54liuyao 已提交
3757
  }
5
54liuyao 已提交
3758
  *pIndex = index + 1;
3759
  return insertNewSessionWindow(pWinInfos, startTs, endTs, index + 1);
5
54liuyao 已提交
3760 3761
}

dengyihao's avatar
dengyihao 已提交
3762 3763
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId,
                                int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted) {
5
54liuyao 已提交
3764
  for (int32_t i = start; i < rows; ++i) {
3765
    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {
5
54liuyao 已提交
3766 3767
      return i - start;
    }
5
54liuyao 已提交
3768
    if (pWinInfo->win.skey > pStartTs[i]) {
5
54liuyao 已提交
3769
      if (pStDeleted && pWinInfo->isOutput) {
H
Haojun Liao 已提交
3770
        SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
3771
        taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
3772 3773
        pWinInfo->isOutput = false;
      }
5
54liuyao 已提交
3774 3775 3776 3777 3778
      pWinInfo->win.skey = pStartTs[i];
    }
    pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pStartTs[i]);
    if (pEndTs) {
      pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pEndTs[i]);
5
54liuyao 已提交
3779 3780 3781 3782 3783
    }
  }
  return rows - start;
}

X
Xiaoyu Wang 已提交
3784
static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,
3785
                                  uint64_t groupId, int32_t numOfOutput, int32_t* rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3786
                                  SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) {
5
54liuyao 已提交
3787 3788
  assert(pWinInfo->win.skey <= pWinInfo->win.ekey);
  // too many time window in query
3789
  int32_t size = taosArrayGetSize(pAggSup->pCurWins);
3790
  if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH && size > MAX_INTERVAL_TIME_WINDOW) {
3791
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
5
54liuyao 已提交
3792
  }
X
Xiaoyu Wang 已提交
3793

5
54liuyao 已提交
3794
  if (pWinInfo->pos.pageId == -1) {
3795
    *pResult = getNewResultRow(pAggSup->pResultBuf, &pAggSup->currentPageId, pAggSup->resultRowSize);
5
54liuyao 已提交
3796 3797 3798 3799 3800 3801 3802 3803
    if (*pResult == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }

    // add a new result set for a new group
    pWinInfo->pos.pageId = (*pResult)->pageId;
    pWinInfo->pos.offset = (*pResult)->offset;
  } else {
3804
    *pResult = getResultRowByPos(pAggSup->pResultBuf, &pWinInfo->pos, true);
5
54liuyao 已提交
3805 3806 3807 3808 3809 3810 3811 3812
    if (!(*pResult)) {
      qError("getResultRowByPos return NULL, TID:%s", GET_TASKID(pTaskInfo));
      return TSDB_CODE_FAILED;
    }
  }

  // set time window for current result
  (*pResult)->win = pWinInfo->win;
3813
  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
5
54liuyao 已提交
3814 3815 3816
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3817 3818 3819
static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup,
                                  SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock,
                                  SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3820
                                  int32_t numOutput, SOperatorInfo* pOperator) {
3821
  SExprSupp*     pSup = &pOperator->exprSupp;
3822 3823
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

X
Xiaoyu Wang 已提交
3824 3825
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId);
  TSKEY*           tsCols = (int64_t*)pColDataInfo->pData;
3826 3827
  int32_t          code = setWindowOutputBuf(pCurWin, pResult, pSup->pCtx, pSDataBlock->info.groupId, numOutput,
                                             pSup->rowEntryInfoOffset, pAggSup, pTaskInfo);
5
54liuyao 已提交
3828 3829 3830
  if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }
5
54liuyao 已提交
3831
  updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false);
3832
  doApplyFunctions(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, pSDataBlock->info.rows, numOutput);
5
54liuyao 已提交
3833 3834 3835
  SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pCurWin->pos.pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pAggSup->pResultBuf, bufPage);
5
54liuyao 已提交
3836 3837 3838
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3839 3840
static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                              SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3841
                              int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3842
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3843
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3844 3845
}

X
Xiaoyu Wang 已提交
3846 3847
static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                                   SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex,
3848
                                   int32_t winRows, int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3849
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3850
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3851 3852
}

5
54liuyao 已提交
3853 3854
int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) {
  SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex);
X
Xiaoyu Wang 已提交
3855
  int32_t            size = taosArrayGetSize(pWinInfos);
5
54liuyao 已提交
3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i);
    if (!isInWindow(pCurWin, pWinInfo->win.skey, gap)) {
      return i - startIndex - 1;
    }
  }

  return size - startIndex - 1;
}

5
54liuyao 已提交
3867
void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId,
3868
                       int32_t numOfOutput, SHashObj* pStUpdated, SHashObj* pStDeleted, SOperatorInfo* pOperator) {
3869
  SExprSupp*     pSup = &pOperator->exprSupp;
3870 3871
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

3872
  SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex);
X
Xiaoyu Wang 已提交
3873
  SResultRow*        pCurResult = NULL;
3874
  setWindowOutputBuf(pCurWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3875
                     &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3876
  num += startIndex + 1;
3877
  ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins));
5
54liuyao 已提交
3878 3879
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < num; i++) {
3880
    SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i);
X
Xiaoyu Wang 已提交
3881
    SResultRow*        pWinResult = NULL;
3882
    setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3883
                       &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3884
    pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey);
3885 3886
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true);
    compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
3887
    taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition));
3888
    if (pWinInfo->isOutput && pStDeleted) {
H
Haojun Liao 已提交
3889
      SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
3890
      taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
3891 3892
      pWinInfo->isOutput = false;
    }
3893
    taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
5
54liuyao 已提交
3894 3895
    SFilePage* tmpPage = getBufPage(pInfo->streamAggSup.pResultBuf, pWinInfo->pos.pageId);
    releaseBufPage(pInfo->streamAggSup.pResultBuf, tmpPage);
5
54liuyao 已提交
3896
  }
5
54liuyao 已提交
3897 3898 3899 3900
  SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pCurWin->pos.pageId);
  ASSERT(num > 0);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
5
54liuyao 已提交
3901 3902
}

3903 3904
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
                                   SHashObj* pStDeleted, bool hasEndTs) {
X
Xiaoyu Wang 已提交
3905
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3906
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3907
  bool                           masterScan = true;
3908
  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
5
54liuyao 已提交
3909
  uint64_t                       groupId = pSDataBlock->info.groupId;
X
Xiaoyu Wang 已提交
3910 3911 3912 3913 3914
  int64_t                        gap = pInfo->gap;
  int64_t                        code = TSDB_CODE_SUCCESS;

  int32_t     step = 1;
  bool        ascScan = true;
5
54liuyao 已提交
3915 3916
  TSKEY*      startTsCols = NULL;
  TSKEY*      endTsCols = NULL;
5
54liuyao 已提交
3917
  SResultRow* pResult = NULL;
X
Xiaoyu Wang 已提交
3918
  int32_t     winRows = 0;
5
54liuyao 已提交
3919

5
54liuyao 已提交
3920 3921 3922 3923 3924 3925
  ASSERT(pSDataBlock->pDataBlock);
  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  startTsCols = (int64_t*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = NULL;
  if (hasEndTs) {
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);
5
54liuyao 已提交
3926
  } else {
5
54liuyao 已提交
3927
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
3928
  }
5
54liuyao 已提交
3929
  endTsCols = (int64_t*)pEndTsCol->pData;
X
Xiaoyu Wang 已提交
3930

5
54liuyao 已提交
3931
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
3932
  for (int32_t i = 0; i < pSDataBlock->info.rows;) {
5
54liuyao 已提交
3933
    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
3934 3935 3936
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
3937
    int32_t            winIndex = 0;
3938
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex);
dengyihao's avatar
dengyihao 已提交
3939 3940
    winRows = updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, groupId, pSDataBlock->info.rows, i, pInfo->gap,
                                      pStDeleted);
3941
    code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
3942
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
3943
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3944
    }
5
54liuyao 已提交
3945

3946
    int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap);
5
54liuyao 已提交
3947
    if (winNum > 0) {
3948
      compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, pStDeleted, pOperator);
5
54liuyao 已提交
3949
    }
5
54liuyao 已提交
3950
    pCurWin->isClosed = false;
5
54liuyao 已提交
3951
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
H
Haojun Liao 已提交
3952 3953
      SWinKey value = {.ts = pCurWin->win.skey, .groupId = groupId};
      code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
5
54liuyao 已提交
3954
      if (code != TSDB_CODE_SUCCESS) {
3955
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3956 3957
      }
      pCurWin->isOutput = true;
5
54liuyao 已提交
3958 3959 3960 3961 3962
    }
    i += winRows;
  }
}

5
54liuyao 已提交
3963
void deleteWindow(SArray* pWinInfos, int32_t index, FDelete fp) {
5
54liuyao 已提交
3964
  ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos));
5
54liuyao 已提交
3965 3966 3967 3968
  if (fp) {
    void* ptr = taosArrayGet(pWinInfos, index);
    fp(ptr);
  }
5
54liuyao 已提交
3969 3970 3971
  taosArrayRemove(pWinInfos, index);
}

3972 3973
static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result,
                                FDelete fp) {
5
54liuyao 已提交
3974
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
3975
  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;
5
54liuyao 已提交
3976
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
3977
  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;
3978
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
3979
  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;
5
54liuyao 已提交
3980
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
3981 3982 3983 3984 3985 3986 3987
    int32_t            winIndex = 0;
    SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex);
    if (!pCurWin) {
      continue;
    }

    do {
3988
      SResultWindowInfo delWin = *pCurWin;
5
54liuyao 已提交
3989
      deleteWindow(pAggSup->pCurWins, winIndex, fp);
5
54liuyao 已提交
3990
      if (result) {
3991 3992
        delWin.groupId = gpDatas[i];
        taosArrayPush(result, &delWin);
5
54liuyao 已提交
3993
      }
3994 3995 3996 3997 3998
      if (winIndex >= taosArrayGetSize(pAggSup->pCurWins)) {
        break;
      }
      pCurWin = taosArrayGet(pAggSup->pCurWins, winIndex);
    } while (pCurWin->win.skey <= endDatas[i]);
5
54liuyao 已提交
3999 4000 4001
  }
}

4002 4003
static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock, int32_t tsIndex,
                                  int32_t numOfOutput, int64_t gap, SArray* result) {
5
54liuyao 已提交
4004
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
4005
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
H
Haojun Liao 已提交
4006 4007
  SColumnInfoData* pGpDataInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*        gpCols = (uint64_t*)pGpDataInfo->pData;
X
Xiaoyu Wang 已提交
4008
  int32_t          step = 0;
5
54liuyao 已提交
4009
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
4010
    int32_t            winIndex = 0;
H
Haojun Liao 已提交
4011
    SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, tsCols[i], INT64_MIN, gpCols[i], gap, &winIndex);
5
54liuyao 已提交
4012
    if (!pCurWin || pCurWin->pos.pageId == -1) {
5
54liuyao 已提交
4013
      // window has been closed.
5
54liuyao 已提交
4014
      step = 1;
5
54liuyao 已提交
4015 4016
      continue;
    }
5
54liuyao 已提交
4017
    step = updateSessionWindowInfo(pCurWin, tsCols, NULL, 0, pBlock->info.rows, i, gap, NULL);
5
54liuyao 已提交
4018
    ASSERT(isInWindow(pCurWin, tsCols[i], gap));
4019
    doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
4020
    if (result) {
4021
      pCurWin->groupId = gpCols[i];
4022 4023
      taosArrayPush(result, pCurWin);
    }
5
54liuyao 已提交
4024 4025 4026
  }
}

5
54liuyao 已提交
4027
static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) {
X
Xiaoyu Wang 已提交
4028
  void*  pData = NULL;
5
54liuyao 已提交
4029
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
4030
  while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) {
5
54liuyao 已提交
4031 4032 4033 4034 4035 4036
    void* key = taosHashGetKey(pData, &keyLen);
    ASSERT(keyLen == sizeof(SResultRowPosition));
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    if (pos == NULL) {
      return TSDB_CODE_QRY_OUT_OF_MEMORY;
    }
H
Haojun Liao 已提交
4037
    pos->groupId = ((SWinKey*)pData)->groupId;
5
54liuyao 已提交
4038
    pos->pos = *(SResultRowPosition*)key;
H
Haojun Liao 已提交
4039
    *(int64_t*)pos->key = ((SWinKey*)pData)->ts;
5
54liuyao 已提交
4040 4041
    taosArrayPush(pUpdated, &pos);
  }
5
54liuyao 已提交
4042
  taosArraySort(pUpdated, resultrowComparAsc);
5
54liuyao 已提交
4043 4044 4045 4046 4047
  return TSDB_CODE_SUCCESS;
}

void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
  blockDataCleanup(pBlock);
4048 4049 4050 4051 4052
  int32_t size = taosHashGetSize(pStDeleted);
  if (size == 0) {
    return;
  }
  blockDataEnsureCapacity(pBlock, size);
5
54liuyao 已提交
4053
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
4054
  while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
H
Haojun Liao 已提交
4055
    SWinKey*         res = *Ite;
4056 4057 4058 4059 4060 4061
    SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
    colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->ts, false);
    SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
    colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->ts, false);
    SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
    colDataAppendNULL(pUidCol, pBlock->info.rows);
5
54liuyao 已提交
4062 4063
    SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
    colDataAppend(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false);
4064 4065 4066 4067
    SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
    colDataAppendNULL(pCalStCol, pBlock->info.rows);
    SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
    colDataAppendNULL(pCalEdCol, pBlock->info.rows);
5
54liuyao 已提交
4068 4069 4070 4071 4072 4073 4074 4075 4076 4077
    pBlock->info.rows += 1;
    if (pBlock->info.rows + 1 >= pBlock->info.capacity) {
      break;
    }
  }
  if ((*Ite) == NULL) {
    taosHashClear(pStDeleted);
  }
}

L
Liu Jicong 已提交
4078
static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t numOfOutput,
4079
                              SOperatorInfo* pOperator, SHashObj* pStUpdated) {
4080
  SExprSupp*     pSup = &pOperator->exprSupp;
4081
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
4082
  int32_t        size = taosArrayGetSize(pWinArray);
4083
  ASSERT(pInfo->pChildren);
4084

4085 4086
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i);
4087
    uint64_t           groupId = pParentWin->groupId;
4088
    int32_t            numOfChildren = taosArrayGetSize(pInfo->pChildren);
4089
    for (int32_t j = 0; j < numOfChildren; j++) {
X
Xiaoyu Wang 已提交
4090
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
4091
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
4092
      SArray*                        pChWins = getWinInfos(&pChInfo->streamAggSup, groupId);
X
Xiaoyu Wang 已提交
4093 4094
      int32_t                        chWinSize = taosArrayGetSize(pChWins);
      int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
4095
      if (index < 0) {
4096
        index = 0;
5
54liuyao 已提交
4097 4098
      }
      for (int32_t k = index; k < chWinSize; k++) {
5
54liuyao 已提交
4099 4100
        SResultWindowInfo* pChWin = taosArrayGet(pChWins, k);
        if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) {
4101 4102 4103 4104 4105 4106
          int32_t            winIndex = 0;
          SResultWindowInfo* pNewParWin =
              getSessionTimeWindow(&pInfo->streamAggSup, pChWin->win.skey, pChWin->win.ekey, groupId, 0, &winIndex);
          SResultRow* pPareResult = NULL;
          setWindowOutputBuf(pNewParWin, &pPareResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
                             &pInfo->streamAggSup, pTaskInfo);
4107
          SResultRow* pChResult = NULL;
5
54liuyao 已提交
4108
          setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
4109
                             pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
4110
          updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pNewParWin->win, true);
4111
          compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
4112 4113 4114 4115 4116 4117

          int32_t winNum = getNumCompactWindow(pInfo->streamAggSup.pCurWins, winIndex, pInfo->gap);
          if (winNum > 0) {
            compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, NULL, pOperator);
          }

5
54liuyao 已提交
4118 4119
          SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
          releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage);
4120 4121 4122 4123 4124 4125

          bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pNewParWin->pos.pageId);
          setBufPageDirty(bufPage, true);
          releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
          SWinKey value = {.ts = pNewParWin->win.skey, .groupId = groupId};
          taosHashPut(pStUpdated, &pNewParWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
5
54liuyao 已提交
4126 4127
        } else if (!pChWin->isClosed) {
          break;
4128 4129 4130 4131 4132 4133
        }
      }
    }
  }
}

X
Xiaoyu Wang 已提交
4134
typedef SResultWindowInfo* (*__get_win_info_)(void*);
5
54liuyao 已提交
4135 4136
SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
5
54liuyao 已提交
4137

4138
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn,
5
54liuyao 已提交
4139
                           bool delete, FDelete fp) {
5
54liuyao 已提交
4140
  // Todo(liuyao) save window to tdb
4141
  void** pIte = NULL;
5
54liuyao 已提交
4142
  size_t keyLen = 0;
4143
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
5
54liuyao 已提交
4144
    uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen);
4145 4146
    SArray*   pWins = (SArray*)(*pIte);
    int32_t   size = taosArrayGetSize(pWins);
4147 4148 4149
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
5
54liuyao 已提交
4150
      if (isCloseWindow(&pSeWin->win, pTwSup)) {
4151 4152
        if (!pSeWin->isClosed) {
          pSeWin->isClosed = true;
5
54liuyao 已提交
4153
          if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {
5
54liuyao 已提交
4154
            int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed);
5
54liuyao 已提交
4155 4156 4157
            if (code != TSDB_CODE_SUCCESS) {
              return code;
            }
4158 4159
            pSeWin->isOutput = true;
          }
5
54liuyao 已提交
4160
          if (delete) {
5
54liuyao 已提交
4161
            deleteWindow(pWins, i, fp);
5
54liuyao 已提交
4162 4163 4164
            i--;
            size = taosArrayGetSize(pWins);
          }
5
54liuyao 已提交
4165
        }
4166
        continue;
5
54liuyao 已提交
4167
      }
4168
      break;
5
54liuyao 已提交
4169 4170 4171 4172 4173
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
4174
static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs, bool delete, FDelete fp) {
5
54liuyao 已提交
4175 4176 4177 4178 4179
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);
    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4180
    closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL, getResWinForSession, delete, fp);
5
54liuyao 已提交
4181 4182 4183
  }
}

4184
int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) {
4185
  void** pIte = NULL;
4186
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
4187
    SArray* pWins = (SArray*)(*pIte);
4188 4189 4190 4191 4192 4193 4194 4195
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (!pSeWin->isClosed) {
        int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed);
        pSeWin->isOutput = true;
      }
5
54liuyao 已提交
4196 4197 4198 4199 4200
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
4201 4202 4203 4204
static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) {
  int32_t size = taosArrayGetSize(pResWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i);
H
Haojun Liao 已提交
4205
    SWinKey            res = {.ts = pWinInfo->win.skey, .groupId = pWinInfo->groupId};
4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218
    taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
  }
}

static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) {
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWin = taosArrayGet(pWins, i);
    taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition));
  }
}

int32_t compareWinKey(void* pKey, void* data, int32_t index) {
L
Liu Jicong 已提交
4219
  SArray*     res = (SArray*)data;
4220
  SResKeyPos* pos = taosArrayGetP(res, index);
L
Liu Jicong 已提交
4221
  SWinKey*    pData = (SWinKey*)pKey;
4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243
  if (pData->ts == *(int64_t*)pos->key) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->ts > *(int64_t*)pos->key) {
    return 1;
  }
  return -1;
}

static void removeSessionDeleteResults(SArray* update, SHashObj* pStDeleted) {
  int32_t size = taosHashGetSize(pStDeleted);
  if (size == 0) {
    return;
  }

  int32_t num = taosArrayGetSize(update);
  for (int32_t i = 0; i < num; i++) {
    SResKeyPos* pos = taosArrayGetP(update, i);
L
Liu Jicong 已提交
4244
    SWinKey     winKey = {.ts = *(int64_t*)pos->key, .groupId = pos->groupId};
4245
    taosHashRemove(pStDeleted, &winKey, sizeof(SWinKey));
5
54liuyao 已提交
4246 4247 4248
  }
}

5
54liuyao 已提交
4249
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
5
54liuyao 已提交
4250
  SExprSupp*                     pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4251
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4252
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
5
54liuyao 已提交
4253 4254 4255 4256
  TSKEY                          maxTs = INT64_MIN;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
4257 4258
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4259
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4260 4261
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
4262
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
4263
    if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
5
54liuyao 已提交
4264 4265
      doSetOperatorCompleted(pOperator);
    }
5
54liuyao 已提交
4266
    printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4267 4268 4269
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
4270 4271
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
4272
  SOperatorInfo* downstream = pOperator->pDownstream[0];
L
Liu Jicong 已提交
4273
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);  // SResKeyPos
5
54liuyao 已提交
4274 4275 4276 4277 4278
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4279
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv");
4280

5
54liuyao 已提交
4281
    if (pBlock->info.type == STREAM_CLEAR) {
X
Xiaoyu Wang 已提交
4282
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
H
Haojun Liao 已提交
4283 4284
      doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, START_TS_COLUMN_INDEX,
                            pOperator->exprSupp.numOfExprs, 0, pWins);
5
54liuyao 已提交
4285 4286
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
X
Xiaoyu Wang 已提交
4287
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
4288
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
L
Liu Jicong 已提交
4289 4290
        doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX,
                              pChildOp->exprSupp.numOfExprs, 0, NULL);
4291
        rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated);
4292 4293
      }
      taosArrayDestroy(pWins);
5
54liuyao 已提交
4294
      continue;
5
54liuyao 已提交
4295 4296 4297
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      // gap must be 0
5
54liuyao 已提交
4298
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL);
5
54liuyao 已提交
4299 4300 4301 4302 4303
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
        // gap must be 0
5
54liuyao 已提交
4304
        doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL);
4305
        rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated);
5
54liuyao 已提交
4306 4307
      }
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
4308
      removeSessionResults(pStUpdated, pWins);
5
54liuyao 已提交
4309 4310
      taosArrayDestroy(pWins);
      continue;
4311
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4312
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
5
54liuyao 已提交
4313
      continue;
5
54liuyao 已提交
4314
    }
5
54liuyao 已提交
4315

5
54liuyao 已提交
4316 4317 4318 4319
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4320
    // the pDataBlock are always the same one, no need to call this again
4321
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4322 4323 4324 4325 4326 4327
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo));
    if (IS_FINAL_OP(pInfo)) {
      int32_t chIndex = getChildIndex(pBlock);
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
4328 4329
        SOperatorInfo* pChildOp =
            createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
5
54liuyao 已提交
4330
        if (!pChildOp) {
4331
          T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4332 4333 4334
        }
        taosArrayPush(pInfo->pChildren, &pChildOp);
      }
4335
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
4336 4337
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true);
4338
    }
5
54liuyao 已提交
4339
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
4340
    maxTs = TMAX(maxTs, pBlock->info.watermark);
5
54liuyao 已提交
4341
  }
5
54liuyao 已提交
4342 4343

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4344 4345
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
4346

4347
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession,
5
54liuyao 已提交
4348 4349
                     pInfo->ignoreExpiredData, NULL);
  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, NULL);
5
54liuyao 已提交
4350
  copyUpdateResult(pStUpdated, pUpdated);
4351
  removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
5
54liuyao 已提交
4352 4353
  taosHashCleanup(pStUpdated);

4354
  finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4355 4356 4357 4358
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4359
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4360 4361 4362
    return pInfo->pDelRes;
  }
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
4363
  printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4364 4365 4366 4367
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
4368
  void** pIte = NULL;
5
54liuyao 已提交
4369
  while ((pIte = taosHashIterate(pInfo->streamAggSup.pResultRows, pIte)) != NULL) {
4370
    SArray* pWins = (SArray*)(*pIte);
5
54liuyao 已提交
4371 4372 4373 4374 4375 4376 4377 4378
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      SResultWindowInfo* pWin = (SResultWindowInfo*)taosArrayGet(pWins, i);
      pWin->pos.pageId = -1;
      pWin->pos.offset = -1;
    }
  }
  clearDiskbasedBuf(pInfo->streamAggSup.pResultBuf);
4379
  pInfo->streamAggSup.currentPageId = -1;
5
54liuyao 已提交
4380 4381 4382 4383 4384 4385 4386
}

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
  TSKEY                          maxTs = INT64_MIN;
  SExprSupp*                     pSup = &pOperator->exprSupp;
4387

5
54liuyao 已提交
4388 4389
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
4390
  }
L
Liu Jicong 已提交
4391

4392
  {
5
54liuyao 已提交
4393 4394
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
    if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
4395
      printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
4396 4397 4398
      return pBInfo->pRes;
    }

4399 4400
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
H
Haojun Liao 已提交
4401
      printDataBlock(pInfo->pDelRes, "semi session");
5
54liuyao 已提交
4402 4403
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
4404

4405 4406
    if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
      pInfo->returnUpdate = false;
5
54liuyao 已提交
4407
      // process the rest of the data
H
Haojun Liao 已提交
4408
      printDataBlock(pInfo->pUpdateRes, "semi session");
5
54liuyao 已提交
4409 4410
      return pInfo->pUpdateRes;
    }
4411 4412

    if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
4413
      clearFunctionContext(&pOperator->exprSupp);
4414 4415 4416 4417 4418
      // semi interval operator clear disk buffer
      clearStreamSessionOperator(pInfo);
      pOperator->status = OP_EXEC_DONE;
      return NULL;
    }
5
54liuyao 已提交
4419 4420 4421 4422 4423 4424 4425 4426 4427
  }

  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  SOperatorInfo* downstream = pOperator->pDownstream[0];
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
4428
      clearSpecialDataBlock(pInfo->pUpdateRes);
4429
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
4430 4431
      break;
    }
H
Haojun Liao 已提交
4432
    printDataBlock(pBlock, "semi session recv");
5
54liuyao 已提交
4433

5
54liuyao 已提交
4434
    if (pBlock->info.type == STREAM_CLEAR) {
5
54liuyao 已提交
4435
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
H
Haojun Liao 已提交
4436
      doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, START_TS_COLUMN_INDEX, pSup->numOfExprs, 0, pWins);
5
54liuyao 已提交
4437 4438
      removeSessionResults(pStUpdated, pWins);
      taosArrayDestroy(pWins);
H
Haojun Liao 已提交
4439
      copyDataBlock(pInfo->pUpdateRes, pBlock);
4440
      pInfo->returnUpdate = true;
5
54liuyao 已提交
4441
      break;
5
54liuyao 已提交
4442 4443
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      // gap must be 0
4444 4445 4446 4447 4448
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL);
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
      removeSessionResults(pStUpdated, pWins);
      taosArrayDestroy(pWins);
5
54liuyao 已提交
4449
      break;
5
54liuyao 已提交
4450 4451 4452 4453 4454
    } else if (pBlock->info.type == STREAM_GET_ALL) {
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
      continue;
    }

5
54liuyao 已提交
4455 4456 4457 4458
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
5
54liuyao 已提交
4459 4460
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
4461
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, NULL, false);
5
54liuyao 已提交
4462 4463 4464 4465
    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
4466
  pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;
4467

5
54liuyao 已提交
4468
  copyUpdateResult(pStUpdated, pUpdated);
4469
  removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
5
54liuyao 已提交
4470
  taosHashCleanup(pStUpdated);
5
54liuyao 已提交
4471

4472 4473
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4474
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
5
54liuyao 已提交
4475
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
4476 4477 4478

  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
  if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
4479
    printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
4480 4481 4482
    return pBInfo->pRes;
  }

4483 4484
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
H
Haojun Liao 已提交
4485
    printDataBlock(pInfo->pDelRes, "semi session");
5
54liuyao 已提交
4486 4487
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
4488

4489 4490
  if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
    pInfo->returnUpdate = false;
5
54liuyao 已提交
4491
    // process the rest of the data
H
Haojun Liao 已提交
4492
    printDataBlock(pInfo->pUpdateRes, "semi session");
5
54liuyao 已提交
4493 4494
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
4495 4496 4497

  pOperator->status = OP_EXEC_DONE;
  return NULL;
5
54liuyao 已提交
4498
}
4499

4500 4501
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                       SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
4502 4503
  int32_t        code = TSDB_CODE_OUT_OF_MEMORY;
  SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo);
4504 4505 4506
  if (pOperator == NULL) {
    goto _error;
  }
4507
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
5
54liuyao 已提交
4508 4509 4510 4511 4512 4513

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
    pInfo->isFinal = true;
    pOperator->name = "StreamSessionFinalAggOperator";
  } else {
    pInfo->isFinal = false;
H
Haojun Liao 已提交
4514
    pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
4515 4516 4517
    blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
    pOperator->name = "StreamSessionSemiAggOperator";
    pOperator->fpSet =
4518 4519
        createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL,
                            destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
4520 4521 4522 4523 4524
  }
  pOperator->operatorType = pPhyNode->type;
  if (numOfChild > 0) {
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
    for (int32_t i = 0; i < numOfChild; i++) {
4525
      SOperatorInfo* pChild = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
5
54liuyao 已提交
4526 4527 4528 4529
      if (pChild == NULL) {
        goto _error;
      }
      taosArrayPush(pInfo->pChildren, &pChild);
4530 4531 4532 4533 4534 4535
    }
  }
  return pOperator;

_error:
  if (pInfo != NULL) {
4536
    destroyStreamSessionAggOperatorInfo(pInfo);
4537 4538 4539 4540 4541 4542
  }

  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5
54liuyao 已提交
4543

4544
void destroyStreamStateOperatorInfo(void* param) {
X
Xiaoyu Wang 已提交
4545
  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;
4546
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
4547
  destroyStateStreamAggSupporter(&pInfo->streamAggSup);
5
54liuyao 已提交
4548 4549 4550 4551
  cleanupGroupResInfo(&pInfo->groupResInfo);
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
4552
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
5
54liuyao 已提交
4553
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
4554
      destroyStreamSessionAggOperatorInfo(pChInfo);
5
54liuyao 已提交
4555 4556 4557 4558
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
5
54liuyao 已提交
4559 4560 4561 4562
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  taosHashCleanup(pInfo->pSeDeleted);
  destroySqlFunctionCtx(pInfo->pDummyCtx, 0);
D
dapan1121 已提交
4563 4564

  taosMemoryFreeClear(param);
5
54liuyao 已提交
4565 4566 4567 4568 4569 4570 4571
}

int64_t getStateWinTsKey(void* data, int32_t index) {
  SStateWindowInfo* pStateWin = taosArrayGet(data, index);
  return pStateWin->winInfo.win.ekey;
}

X
Xiaoyu Wang 已提交
4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583
SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4584 4585 4586 4587 4588 4589 4590 4591
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayPush(pWinInfos, &win);
}

X
Xiaoyu Wang 已提交
4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603
SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayInsert(pWinInfos, index, &win);
}

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {
  if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) {
    return true;
  }
  return false;
}

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {
  return pKeyData && compareVal(pKeyData, &pWin->stateKey);
}

4623 4624 4625
SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
X
Xiaoyu Wang 已提交
4626 4627
  int32_t           size = taosArrayGetSize(pWinInfos);
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index + 1;
      return pWin;
    }
  }
  *pIndex = 0;
  return NULL;
}

4648 4649
SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
                                 SColumn* pCol, int32_t* pIndex) {
4650 4651
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
5
54liuyao 已提交
4652 4653 4654 4655 4656
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    *pIndex = 0;
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
X
Xiaoyu Wang 已提交
4657
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
  *pIndex = index + 1;
  return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol);
}

H
Haojun Liao 已提交
4691 4692 4693
int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, uint64_t groupId,
                              SColumnInfoData* pKeyCol, int32_t rows, int32_t start, bool* allEqual,
                              SHashObj* pSeDeleted) {
5
54liuyao 已提交
4694 4695 4696 4697 4698
  *allEqual = true;
  SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex);
  for (int32_t i = start; i < rows; ++i) {
    char* pKeyData = colDataGetData(pKeyCol, i);
    if (!isTsInWindow(pWinInfo, pTs[i])) {
X
Xiaoyu Wang 已提交
4699
      if (isEqualStateKey(pWinInfo, pKeyData)) {
5
54liuyao 已提交
4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712
        int32_t size = taosArrayGetSize(pWinInfos);
        if (winIndex + 1 < size) {
          SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1);
          // ts belongs to the next window
          if (pTs[i] >= pNextWin->winInfo.win.skey) {
            return i - start;
          }
        }
      } else {
        return i - start;
      }
    }
    if (pWinInfo->winInfo.win.skey > pTs[i]) {
H
Haojun Liao 已提交
4713 4714
      if (pSeDeleted && pWinInfo->winInfo.isOutput) {
        SWinKey res = {.ts = pWinInfo->winInfo.win.skey, .groupId = groupId};
4715
        taosHashPut(pSeDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727
        pWinInfo->winInfo.isOutput = false;
      }
      pWinInfo->winInfo.win.skey = pTs[i];
    }
    pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]);
    if (!isEqualStateKey(pWinInfo, pKeyData)) {
      *allEqual = false;
    }
  }
  return rows - start;
}

H
Haojun Liao 已提交
4728 4729 4730 4731
static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SHashObj* pSeUpdated,
                                SHashObj* pSeDeleted) {
  SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupColInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
X
Xiaoyu Wang 已提交
4732 4733 4734
  TSKEY*           tsCol = (TSKEY*)pTsColInfo->pData;
  bool             allEqual = false;
  int32_t          step = 1;
H
Haojun Liao 已提交
4735
  uint64_t*        gpCol = (uint64_t*)pGroupColInfo->pData;
5
54liuyao 已提交
4736
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
4737
    int32_t           winIndex = 0;
H
Haojun Liao 已提交
4738
    SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], gpCol[i], &winIndex);
5
54liuyao 已提交
4739 4740 4741
    if (!pCurWin) {
      continue;
    }
H
Haojun Liao 已提交
4742
    updateSessionWindowInfo(&pCurWin->winInfo, tsCol, NULL, 0, pBlock->info.rows, i, 0, NULL);
5
54liuyao 已提交
4743
    taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
5
54liuyao 已提交
4744
    deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
5
54liuyao 已提交
4745 4746 4747
  }
}

X
Xiaoyu Wang 已提交
4748 4749 4750
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated,
                                 SHashObj* pStDeleted) {
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
4751
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4752
  bool                         masterScan = true;
4753
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
X
Xiaoyu Wang 已提交
4754 4755 4756 4757 4758 4759 4760
  int64_t                      groupId = pSDataBlock->info.groupId;
  int64_t                      code = TSDB_CODE_SUCCESS;
  int32_t                      step = 1;
  bool                         ascScan = true;
  TSKEY*                       tsCols = NULL;
  SResultRow*                  pResult = NULL;
  int32_t                      winRows = 0;
5
54liuyao 已提交
4761
  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
4762 4763
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
4764
  } else {
X
Xiaoyu Wang 已提交
4765
    return;
5
54liuyao 已提交
4766
  }
L
Liu Jicong 已提交
4767

5
54liuyao 已提交
4768
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
4769
  blockDataEnsureCapacity(pAggSup->pScanBlock, pSDataBlock->info.rows);
L
Liu Jicong 已提交
4770
  SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
X
Xiaoyu Wang 已提交
4771
  for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
5
54liuyao 已提交
4772
    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
4773 4774 4775
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
4776 4777 4778
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
    bool              allEqual = true;
H
Haojun Liao 已提交
4779 4780 4781
    SStateWindowInfo* pCurWin = getStateWindow(pAggSup, tsCols[i], groupId, pKeyData, &pInfo->stateCol, &winIndex);
    winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, groupId, pKeyColInfo, pSDataBlock->info.rows,
                                    i, &allEqual, pStDeleted);
5
54liuyao 已提交
4782
    if (!allEqual) {
4783
      uint64_t uid = 0;
L
Liu Jicong 已提交
4784
      appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, &uid, &groupId);
5
54liuyao 已提交
4785
      taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
5
54liuyao 已提交
4786
      deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
5
54liuyao 已提交
4787 4788
      continue;
    }
4789
    code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
4790
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
4791
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4792 4793 4794
    }
    pCurWin->winInfo.isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
H
Haojun Liao 已提交
4795 4796
      SWinKey value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId};
      code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
5
54liuyao 已提交
4797
      if (code != TSDB_CODE_SUCCESS) {
4798
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809
      }
      pCurWin->winInfo.isOutput = true;
    }
  }
}

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

4810
  SExprSupp*                   pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4811
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4812
  SOptrBasicInfo*              pBInfo = &pInfo->binfo;
L
Liu Jicong 已提交
4813
  int64_t                      maxTs = INT64_MIN;
5
54liuyao 已提交
4814 4815 4816
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4817
      printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4818 4819
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
4820
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
4821
    if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
5
54liuyao 已提交
4822 4823
      doSetOperatorCompleted(pOperator);
    }
5
54liuyao 已提交
4824
    printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4825 4826 4827
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
4828 4829
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
4830
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
4831
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
4832 4833 4834 4835 4836
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4837
    printDataBlock(pBlock, "single state recv");
4838

5
54liuyao 已提交
4839
    if (pBlock->info.type == STREAM_CLEAR) {
H
Haojun Liao 已提交
4840
      doClearStateWindows(&pInfo->streamAggSup, pBlock, pSeUpdated, pInfo->pSeDeleted);
5
54liuyao 已提交
4841
      continue;
4842
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
4843
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
5
54liuyao 已提交
4844
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo);
4845
      copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
4846
      removeSessionResults(pSeUpdated, pWins);
4847 4848
      taosArrayDestroy(pWins);
      continue;
4849
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4850
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState);
5
54liuyao 已提交
4851
      continue;
5
54liuyao 已提交
4852
    }
4853

5
54liuyao 已提交
4854 4855 4856 4857
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4858
    // the pDataBlock are always the same one, no need to call this again
4859
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4860
    doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
4861
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
4862
  }
4863
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4864 4865
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
X
Xiaoyu Wang 已提交
4866

4867
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState,
5
54liuyao 已提交
4868 4869
                     pInfo->ignoreExpiredData, destroyStateWinInfo);
  // closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, destroyStateWinInfo);
5
54liuyao 已提交
4870
  copyUpdateResult(pSeUpdated, pUpdated);
5
54liuyao 已提交
4871 4872
  taosHashCleanup(pSeUpdated);

4873 4874
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4875 4876 4877 4878
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4879
    printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4880 4881
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
4882
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
4883
  printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4884 4885 4886
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

4887 4888 4889 4890
int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo));
}

X
Xiaoyu Wang 已提交
4891 4892 4893 4894 4895 4896
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
  SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
4897
  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
4898

X
Xiaoyu Wang 已提交
4899 4900
  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
4901 4902 4903 4904
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

4905 4906
  SExprSupp* pSup = &pOperator->exprSupp;

X
Xiaoyu Wang 已提交
4907
  int32_t    numOfCols = 0;
5
54liuyao 已提交
4908 4909 4910
  SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
4911
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
4912 4913 4914 4915 4916 4917 4918 4919 4920
  if (pStateNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

4921
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
X
Xiaoyu Wang 已提交
4922 4923
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
5
54liuyao 已提交
4924 4925
      .calTrigger = pStateNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
4926
      .minTs = INT64_MAX,
X
Xiaoyu Wang 已提交
4927
  };
5
54liuyao 已提交
4928
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
4929

4930
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
4931 4932 4933 4934
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

4935
  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
4936 4937 4938 4939 4940 4941 4942 4943 4944
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }

4945
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
4946 4947 4948 4949 4950
  pInfo->primaryTsIndex = tsSlotId;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
H
Haojun Liao 已提交
4951
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
5
54liuyao 已提交
4952
  pInfo->pChildren = NULL;
5
54liuyao 已提交
4953
  pInfo->ignoreExpiredData = pStateNode->window.igExpired;
5
54liuyao 已提交
4954 4955

  pOperator->name = "StreamStateAggOperator";
4956
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
5
54liuyao 已提交
4957 4958 4959 4960
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
4961 4962
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL,
                                         destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
L
Liu Jicong 已提交
4963 4964
  initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType,
                 pInfo->primaryTsIndex);
5
54liuyao 已提交
4965 4966 4967 4968 4969 4970 4971
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
  return pOperator;

_error:
4972
  destroyStreamStateOperatorInfo(pInfo);
5
54liuyao 已提交
4973 4974 4975 4976
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
4977

4978
void destroyMAIOperatorInfo(void* param) {
4979
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param;
4980
  destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo);
D
dapan1121 已提交
4981
  taosMemoryFreeClear(param);
4982 4983
}

4984
static SResultRow* doSetSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, SAggSupporter* pSup) {
H
Haojun Liao 已提交
4985 4986
  SResultRow* pResult = getNewResultRow(pSup->pResultBuf, &pSup->currentPageId, pSup->resultRowSize);
  pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
4987 4988
  return pResult;
}
4989

4990 4991 4992 4993 4994 4995 4996 4997
static int32_t setSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, SResultRow** pResult,
                                       SExprSupp* pExprSup, SAggSupporter* pAggSup) {
  if (*pResult == NULL) {
    *pResult = doSetSingleOutputTupleBuf(pResultRowInfo, pAggSup);
    if (*pResult == NULL) {
      return terrno;
    }
  }
4998

4999
  // set time window for current result
5000 5001
  (*pResult)->win = (*win);
  setResultRowInitCtx((*pResult), pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
5002
  return TSDB_CODE_SUCCESS;
5003 5004
}

5005
static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
5006
                                          SSDataBlock* pBlock, SSDataBlock* pResultBlock) {
5007
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
D
dapan1121 已提交
5008
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
5009 5010

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
5011
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
5012
  SInterval*     pInterval = &iaInfo->interval;
5013

5
54liuyao 已提交
5014 5015
  int32_t  startPos = 0;
  int64_t* tsCols = extractTsCol(pBlock, iaInfo);
5016

5017 5018
  TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols);

5019 5020
  // there is an result exists
  if (miaInfo->curTs != INT64_MIN) {
5021
    if (ts != miaInfo->curTs) {
5022
      finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo);
5023
      resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow));
5024
      miaInfo->curTs = ts;
5025
    }
5026 5027
  } else {
    miaInfo->curTs = ts;
5028 5029 5030
  }

  STimeWindow win = {0};
5031
  win.skey = miaInfo->curTs;
5032
  win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
5033

5
54liuyao 已提交
5034
  int32_t ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
5035 5036
  if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) {
    T_LONG_JMP(pTaskInfo->env, ret);
5037 5038
  }

5039 5040
  int32_t currPos = startPos;

5041
  STimeWindow currWin = win;
5042
  while (++currPos < pBlock->info.rows) {
5043
    if (tsCols[currPos] == miaInfo->curTs) {
5044
      continue;
5045 5046 5047
    }

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
5048
    doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
5049
                     pBlock->info.rows, pSup->numOfExprs);
5050

5051
    finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo);
5052
    resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow));
5053
    miaInfo->curTs = tsCols[currPos];
5054

5055
    currWin.skey = miaInfo->curTs;
5056
    currWin.ekey = taosTimeAdd(currWin.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
5057 5058

    startPos = currPos;
5
54liuyao 已提交
5059
    ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
5060 5061
    if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) {
      T_LONG_JMP(pTaskInfo->env, ret);
5062
    }
5063 5064

    miaInfo->curTs = currWin.skey;
5065
  }
5066

5067
  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
5068
  doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
5069
                   pBlock->info.rows, pSup->numOfExprs);
5070 5071
}

5072 5073 5074 5075
static void cleanupAfterGroupResultGen(SMergeAlignedIntervalAggOperatorInfo* pMiaInfo, SSDataBlock* pRes) {
  pRes->info.groupId = pMiaInfo->groupId;
  pMiaInfo->curTs = INT64_MIN;
  pMiaInfo->groupId = 0;
5076 5077
}

5078
static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
S
shenglian zhou 已提交
5079
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
5080

5081 5082
  SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info;
  SIntervalAggOperatorInfo*             pIaInfo = pMiaInfo->intervalAggOperatorInfo;
5083

5084 5085 5086 5087 5088
  SExprSupp*      pSup = &pOperator->exprSupp;
  SSDataBlock*    pRes = pIaInfo->binfo.pRes;
  SResultRowInfo* pResultRowInfo = &pIaInfo->binfo.resultRowInfo;
  SOperatorInfo*  downstream = pOperator->pDownstream[0];
  int32_t         scanFlag = MAIN_SCAN;
5089

5090 5091
  while (1) {
    SSDataBlock* pBlock = NULL;
5092
    if (pMiaInfo->prefetchedBlock == NULL) {
5093 5094
      pBlock = downstream->fpSet.getNextFn(downstream);
    } else {
5095 5096
      pBlock = pMiaInfo->prefetchedBlock;
      pMiaInfo->prefetchedBlock = NULL;
5097

5098
      pMiaInfo->groupId = pBlock->info.groupId;
5099
    }
5100

5101
    // no data exists, all query processing is done
5102
    if (pBlock == NULL) {
5103 5104 5105
      // close last unclosed time window
      if (pMiaInfo->curTs != INT64_MIN) {
        finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo);
5106 5107
        resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow));
        cleanupAfterGroupResultGen(pMiaInfo, pRes);
5108
      }
5109

5110 5111
      doSetOperatorCompleted(pOperator);
      break;
5112
    }
5113

H
Haojun Liao 已提交
5114 5115 5116 5117 5118 5119 5120 5121 5122
    if (pMiaInfo->groupId == 0) {
      if (pMiaInfo->groupId != pBlock->info.groupId) {
        pMiaInfo->groupId = pBlock->info.groupId;
      }
    } else {
      if (pMiaInfo->groupId != pBlock->info.groupId) {
        // if there are unclosed time window, close it firstly.
        ASSERT(pMiaInfo->curTs != INT64_MIN);
        finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo);
5123
        resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow));
H
Haojun Liao 已提交
5124

5125 5126
        pMiaInfo->prefetchedBlock = pBlock;
        cleanupAfterGroupResultGen(pMiaInfo, pRes);
H
Haojun Liao 已提交
5127
        break;
5
54liuyao 已提交
5128
      } else {
H
Haojun Liao 已提交
5129 5130
        // continue
      }
5131
    }
5132

5133 5134 5135
    getTableScanInfo(pOperator, &pIaInfo->inputOrder, &scanFlag);
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pIaInfo->inputOrder, scanFlag, true);
    doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes);
5136

5137
    doFilter(pMiaInfo->pCondition, pRes, NULL);
5138 5139 5140
    if (pRes->info.rows >= pOperator->resultInfo.capacity) {
      break;
    }
5141
  }
5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156
}

static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info;
  SIntervalAggOperatorInfo*             iaInfo = pMiaInfo->intervalAggOperatorInfo;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);

  if (iaInfo->binfo.mergeResultBlock) {
dengyihao's avatar
dengyihao 已提交
5157
    while (1) {
5158
      if (pOperator->status == OP_EXEC_DONE) {
5159 5160
        break;
      }
5161

5162
      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
5163 5164 5165
        break;
      }

5166 5167 5168 5169
      doMergeAlignedIntervalAgg(pOperator);
    }
  } else {
    doMergeAlignedIntervalAgg(pOperator);
5170 5171 5172 5173 5174 5175 5176
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

5177
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode,
5178
                                                      SExecTaskInfo* pTaskInfo) {
5179
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo));
5180
  SOperatorInfo*                        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5181 5182 5183 5184
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

D
dapan1121 已提交
5185 5186 5187 5188 5189
  miaInfo->intervalAggOperatorInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
  if (miaInfo->intervalAggOperatorInfo == NULL) {
    goto _error;
  }

5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200
  int32_t      num = 0;
  SExprInfo*   pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pNode->window.node.pOutputDataBlockDesc);

  SInterval interval = {.interval = pNode->interval,
                        .sliding = pNode->sliding,
                        .intervalUnit = pNode->intervalUnit,
                        .slidingUnit = pNode->slidingUnit,
                        .offset = pNode->offset,
                        .precision = ((SColumnNode*)pNode->window.pTspk)->node.resType.precision};

D
dapan1121 已提交
5201
  SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
5202
  SExprSupp*                pSup = &pOperator->exprSupp;
5203

5204
  miaInfo->pCondition = pNode->window.node.pConditions;
L
Liu Jicong 已提交
5205 5206 5207 5208 5209
  miaInfo->curTs = INT64_MIN;
  iaInfo->win = pTaskInfo->window;
  iaInfo->inputOrder = TSDB_ORDER_ASC;
  iaInfo->interval = interval;
  iaInfo->execModel = pTaskInfo->execModel;
5210 5211
  iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId;
  iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock;
5212 5213

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
5214
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5215

5216
  int32_t code = initAggInfo(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
5217 5218 5219
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
5220

5221
  initBasicInfo(&iaInfo->binfo, pResBlock);
5222
  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
5223

5224
  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, iaInfo);
5225
  if (iaInfo->timeWindowInterpo) {
5226
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
5227 5228
  }

5229
  initResultRowInfo(&iaInfo->binfo.resultRowInfo);
5230
  blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity);
5231

L
Liu Jicong 已提交
5232
  pOperator->name = "TimeMergeAlignedIntervalAggOperator";
5233
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL;
L
Liu Jicong 已提交
5234 5235 5236 5237
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = miaInfo;
5238

5239
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL,
5240
                                         destroyMAIOperatorInfo, NULL, NULL, NULL);
5241 5242 5243 5244 5245 5246 5247 5248 5249

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5250
  destroyMAIOperatorInfo(miaInfo);
5251 5252 5253 5254
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5255 5256 5257 5258 5259

//=====================================================================================================================
// merge interval operator
typedef struct SMergeIntervalAggOperatorInfo {
  SIntervalAggOperatorInfo intervalAggOperatorInfo;
L
Liu Jicong 已提交
5260 5261 5262 5263 5264 5265
  SList*                   groupIntervals;
  SListIter                groupIntervalsIter;
  bool                     hasGroupId;
  uint64_t                 groupId;
  SSDataBlock*             prefetchedBlock;
  bool                     inputBlocksFinished;
5266 5267
} SMergeIntervalAggOperatorInfo;

S
slzhou 已提交
5268
typedef struct SGroupTimeWindow {
L
Liu Jicong 已提交
5269
  uint64_t    groupId;
S
slzhou 已提交
5270 5271 5272
  STimeWindow window;
} SGroupTimeWindow;

5273
void destroyMergeIntervalOperatorInfo(void* param) {
5274
  SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
S
slzhou 已提交
5275
  tdListFree(miaInfo->groupIntervals);
5276
  destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo);
5277

D
dapan1121 已提交
5278
  taosMemoryFreeClear(param);
5279 5280
}

L
Liu Jicong 已提交
5281 5282
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win,
                                    SSDataBlock* pResultBlock) {
5283 5284 5285
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
5286
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5287 5288 5289
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId);
L
Liu Jicong 已提交
5290 5291
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(
      iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
5292
  ASSERT(p1 != NULL);
5
54liuyao 已提交
5293
  //  finalizeResultRows(iaInfo->aggSup.pResultBuf, p1, pResultBlock, pTaskInfo);
5294
  tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
5295 5296 5297
  return TSDB_CODE_SUCCESS;
}

5298 5299 5300 5301
static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock,
                                        STimeWindow* newWin) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
5302
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5303

S
slzhou 已提交
5304 5305
  SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin};
  tdListAppend(miaInfo->groupIntervals, &groupTimeWindow);
5306

S
slzhou 已提交
5307 5308 5309 5310 5311
  SListIter iter = {0};
  tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD);
  SListNode* listNode = NULL;
  while ((listNode = tdListNext(&iter)) != NULL) {
    SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
L
Liu Jicong 已提交
5312
    if (prevGrpWin->groupId != tableGroupId) {
S
slzhou 已提交
5313 5314
      continue;
    }
5315

S
slzhou 已提交
5316
    STimeWindow* prevWin = &prevGrpWin->window;
H
Haojun Liao 已提交
5317
    if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) {
5
54liuyao 已提交
5318
      //      finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
S
slzhou 已提交
5319 5320
      tdListPopNode(miaInfo->groupIntervals, listNode);
    }
5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337
  }

  return 0;
}

static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                                   int32_t scanFlag, SSDataBlock* pResultBlock) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*     pExprSup = &pOperatorInfo->exprSupp;

  int32_t     startPos = 0;
  int32_t     numOfOutput = pExprSup->numOfExprs;
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
5338
  bool        ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5339 5340 5341
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

5342 5343
  STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval,
                                        iaInfo->inputOrder);
5344 5345 5346 5347 5348

  int32_t ret =
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                             numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
5349
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5350 5351 5352 5353
  }

  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
5354
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5355 5356 5357 5358
  ASSERT(forwardRows > 0);

  // prev time window not interpolation yet.
  if (iaInfo->timeWindowInterpo) {
5359
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
5360 5361 5362 5363 5364 5365
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);

    // restore current time window
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                                 numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS) {
5366
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5367 5368 5369 5370 5371 5372 5373
    }

    // window start key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup);
  }

  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true);
5374 5375
  doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
                   pBlock->info.rows, numOfOutput);
5376 5377 5378 5379 5380 5381 5382 5383
  doCloseWindow(pResultRowInfo, iaInfo, pResult);

  // output previous interval results after this interval (&win) is closed
  outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win);

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = forwardRows - 1 + startPos;
5384 5385
    startPos =
        getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->inputOrder);
5386 5387 5388 5389 5390 5391 5392 5393 5394
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
    int32_t code =
        setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                               pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
5395
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5396 5397 5398 5399
    }

    ekey = ascScan ? nextWin.ekey : nextWin.skey;
    forwardRows =
5400
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5401 5402 5403 5404 5405

    // window start(end) key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup);

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true);
5406 5407
    doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pBlock->info.rows, numOfOutput);
5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447
    doCloseWindow(pResultRowInfo, iaInfo, pResult);

    // output previous interval results after this interval (&nextWin) is closed
    outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin);
  }

  if (iaInfo->timeWindowInterpo) {
    saveDataBlockLastRow(iaInfo->pPrevValues, pBlock, iaInfo->pInterpCols);
  }
}

static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExprSupp*                     pExpSupp = &pOperator->exprSupp;

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);

  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
        miaInfo->groupId = pBlock->info.groupId;
        miaInfo->prefetchedBlock = NULL;
      }

      if (pBlock == NULL) {
S
slzhou 已提交
5448
        tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD);
5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460
        miaInfo->inputBlocksFinished = true;
        break;
      }

      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
        miaInfo->groupId = pBlock->info.groupId;
      } else if (miaInfo->groupId != pBlock->info.groupId) {
        miaInfo->prefetchedBlock = pBlock;
        break;
      }

5461 5462
      getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag);
      setInputDataBlock(pOperator, pExpSupp->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true);
5463 5464 5465 5466 5467 5468 5469 5470
      doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);

      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
        break;
      }
    }

    pRes->info.groupId = miaInfo->groupId;
5471 5472 5473
  }

  if (miaInfo->inputBlocksFinished) {
S
slzhou 已提交
5474
    SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter);
5475

S
slzhou 已提交
5476 5477
    if (listNode != NULL) {
      SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data);
5
54liuyao 已提交
5478
      //      finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
S
slzhou 已提交
5479
      pRes->info.groupId = grpWin->groupId;
5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491
    }
  }

  if (pRes->info.rows == 0) {
    doSetOperatorCompleted(pOperator);
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

5492 5493 5494
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode,
                                               SExecTaskInfo* pTaskInfo) {
  SMergeIntervalAggOperatorInfo* pMergeIntervalInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
5495
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5496
  if (pMergeIntervalInfo == NULL || pOperator == NULL) {
5497 5498 5499
    goto _error;
  }

5500 5501 5502 5503 5504 5505 5506 5507 5508 5509
  int32_t      num = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pIntervalPhyNode->window.node.pOutputDataBlockDesc);

  SInterval interval = {.interval = pIntervalPhyNode->interval,
                        .sliding = pIntervalPhyNode->sliding,
                        .intervalUnit = pIntervalPhyNode->intervalUnit,
                        .slidingUnit = pIntervalPhyNode->slidingUnit,
                        .offset = pIntervalPhyNode->offset,
                        .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
5510

5511
  pMergeIntervalInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow));
5512

5513
  SIntervalAggOperatorInfo* pIntervalInfo = &pMergeIntervalInfo->intervalAggOperatorInfo;
L
Liu Jicong 已提交
5514
  pIntervalInfo->win = pTaskInfo->window;
5515
  pIntervalInfo->inputOrder = TSDB_ORDER_ASC;
L
Liu Jicong 已提交
5516 5517
  pIntervalInfo->interval = interval;
  pIntervalInfo->execModel = pTaskInfo->execModel;
5518 5519
  pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock;
  pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
5520 5521 5522 5523

  SExprSupp* pExprSupp = &pOperator->exprSupp;

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
5524
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5525

5526
  int32_t code = initAggInfo(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
5527 5528 5529
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
5530

5531 5532
  initBasicInfo(&pIntervalInfo->binfo, pResBlock);
  initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win);
5533

5534 5535
  pIntervalInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, num, pIntervalInfo);
  if (pIntervalInfo->timeWindowInterpo) {
5536
    pIntervalInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
5537
    if (pIntervalInfo->binfo.resultRowInfo.openWindow == NULL) {
5538 5539 5540 5541
      goto _error;
    }
  }

5542
  initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo);
5543

L
Liu Jicong 已提交
5544
  pOperator->name = "TimeMergeIntervalAggOperator";
5545
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL;
L
Liu Jicong 已提交
5546 5547 5548 5549
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pMergeIntervalInfo;
5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL,
                                         destroyMergeIntervalOperatorInfo, NULL, NULL, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5562
  destroyMergeIntervalOperatorInfo(pMergeIntervalInfo);
5563 5564 5565
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
5566
}
5567

5568 5569
static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                                    int32_t scanFlag, SHashObj* pUpdatedMap) {
5570 5571 5572 5573 5574
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;

5575 5576
  int32_t          startPos = 0;
  int32_t          numOfOutput = pSup->numOfExprs;
5577
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
5578 5579 5580 5581 5582
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
  uint64_t         tableGroupId = pBlock->info.groupId;
  bool             ascScan = true;
  TSKEY            ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow*      pResult = NULL;
5583

5584 5585
  STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
  int32_t     ret = TSDB_CODE_SUCCESS;
5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645
  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
    }
  }

  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
  ASSERT(forwardRows > 0);

  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
  }

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = forwardRows - 1 + startPos;
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);
    if (startPos < 0) {
      break;
    }
    if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
      ekey = ascScan ? nextWin.ekey : nextWin.skey;
      forwardRows =
          getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
      continue;
    }

    // null data, failed to allocate more memory buffer
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
    }

    ekey = ascScan ? nextWin.ekey : nextWin.skey;
    forwardRows =
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
  }
}

5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686
static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
                                     SHashObj* pUpdatedMap) {
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;

  SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*  pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*      pSup = &pOperatorInfo->exprSupp;
  int32_t         numOfOutput = pSup->numOfExprs;
  int32_t         step = 1;
  TSKEY*          tsCols = NULL;
  SResultRow*     pResult = NULL;
  int32_t         forwardRows = 0;
  int32_t         aa = 4;

  ASSERT(pSDataBlock->pDataBlock != NULL);
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  tsCols = (int64_t*)pColDataInfo->pData;

  int32_t     startPos = 0;
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
  STimeWindow nextWin =
      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
  while (1) {
    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
    if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
      if (startPos < 0) {
        break;
      }
      continue;
    }

    int32_t code = setOutputBuf(&nextWin, &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset,
                                &pInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
                                           TSDB_ORDER_ASC);
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
5
54liuyao 已提交
5687
      saveWinResultInfo(pResult->win.skey, tableGroupId, pUpdatedMap);
5688 5689 5690 5691 5692 5693 5694 5695
    }
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pSDataBlock->info.rows, numOfOutput);
    SWinKey key = {
        .ts = nextWin.skey,
        .groupId = tableGroupId,
    };
5
54liuyao 已提交
5696
    saveOutputBuf(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize);
5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722
    releaseOutputBuf(pTaskInfo, &key, pResult);
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
    startPos =
        getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);
    if (startPos < 0) {
      break;
    }
  }
}

void doBuildResult(SOperatorInfo* pOperator, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  // set output datablock version
  pBlock->info.version = pTaskInfo->version;

  blockDataCleanup(pBlock);
  if (!hasRemainResults(pGroupResInfo)) {
    return;
  }

  // clear the existed group id
  pBlock->info.groupId = 0;
  buildDataBlockFromGroupRes(pTaskInfo, pBlock, &pOperator->exprSupp, pGroupResInfo);
}

5723 5724
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
5725 5726
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
  int64_t                      maxTs = INT64_MIN;
5
54liuyao 已提交
5727
  int64_t                      minTs = INT64_MAX;
5728
  SExprSupp*                   pSup = &pOperator->exprSupp;
5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows > 0) {
      printDataBlock(pInfo->pDelRes, "single interval");
      return pInfo->pDelRes;
    }

5
54liuyao 已提交
5741 5742
    doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo);
    // doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767
    if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
      pOperator->status = OP_EXEC_DONE;
      qDebug("===stream===single interval is done");
      freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
    }
    printDataBlock(pInfo->binfo.pRes, "single interval");
    return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];

  SArray*    pUpdated = taosArrayInit(4, POINTER_BYTES);  // SResKeyPos
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*  pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);

  SStreamState* pState = pTaskInfo->streamInfo.pState;

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
    printDataBlock(pBlock, "single interval recv");

    if (pBlock->info.type == STREAM_CLEAR) {
5
54liuyao 已提交
5768
      doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, NULL, NULL);
5769 5770
      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
      continue;
5771
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
5
54liuyao 已提交
5772 5773 5774 5775
      // doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, pInfo->pDelWins, &pInfo->interval,
      //                               pUpdatedMap);
      doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pInfo->pDelWins,
                      pUpdatedMap);
5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800
      continue;
    } else if (pBlock->info.type == STREAM_GET_ALL) {
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
      continue;
    }

    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {
      // set input version
      pTaskInfo->version = pBlock->info.version;
    }

    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }

    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
    // caller. Note that all the time window are not close till now.
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
    if (pInfo->invertible) {
      setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type);
    }

    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
5801
    minTs = TMIN(minTs, pBlock->info.window.skey);
5
54liuyao 已提交
5802
    // doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap);
5803
    // new disc buf
5
54liuyao 已提交
5804
    doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
5805 5806
  }
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
5807
  pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs);
5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838

#if 0
  if (pState) {
    printf(">>>>>>>> stream read backend\n");
    SWinKey key = {
        .ts = 1,
        .groupId = 2,
    };
    char*   val = NULL;
    int32_t sz;
    if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) {
      ASSERT(0);
    }
    printf("stream read %s %d\n", val, sz);
    streamFreeVal(val);

    SStreamStateCur* pCur = streamStateGetCur(pState, &key);
    ASSERT(pCur);
    while (streamStateCurNext(pState, pCur) == 0) {
      SWinKey     key1;
      const void* val1;
      if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) {
        break;
      }
      printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz);
    }
    streamStateFreeCur(pCur);
  }
#endif

  pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
5839 5840
  closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap,
                            pOperator);
5841 5842 5843 5844 5845 5846 5847

  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosArraySort(pUpdated, resultrowComparAsc);

5848
  // new disc buf
5
54liuyao 已提交
5849 5850
  // finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated,
  // pSup->rowEntryInfoOffset);
5851 5852 5853 5854 5855 5856 5857 5858 5859 5860
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
  taosHashCleanup(pUpdatedMap);
  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows > 0) {
    printDataBlock(pInfo->pDelRes, "single interval");
    return pInfo->pDelRes;
  }

5
54liuyao 已提交
5861
  // doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5862
  // new disc buf
5
54liuyao 已提交
5863
  doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo);
5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884
  printDataBlock(pInfo->binfo.pRes, "single interval");
  return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}

void destroyStreamIntervalOperatorInfo(void* param) {
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;
  cleanupBasicInfo(&pInfo->binfo);
  cleanupAggSup(&pInfo->aggSup);
  pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages);

  pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins);
  pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);

  cleanupGroupResInfo(&pInfo->groupResInfo);
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  taosMemoryFreeClear(param);
}

SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
5885
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5886 5887 5888 5889 5890
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

5891 5892
  int32_t    numOfCols = 0;
  SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5893 5894
  ASSERT(numOfCols > 0);
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906
  SInterval    interval = {
         .interval = pIntervalPhyNode->interval,
         .sliding = pIntervalPhyNode->sliding,
         .intervalUnit = pIntervalPhyNode->intervalUnit,
         .slidingUnit = pIntervalPhyNode->slidingUnit,
         .offset = pIntervalPhyNode->offset,
         .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,
  };
  STimeWindowAggSupp twAggSupp = {
      .waterMark = pIntervalPhyNode->window.watermark,
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
5907
      .minTs = INT64_MAX,
5
54liuyao 已提交
5908
      .deleteMark = INT64_MAX,
5909
  };
5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925
  ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY);
  pOperator->pTaskInfo = pTaskInfo;
  pInfo->interval = interval;
  pInfo->twAggSup = twAggSupp;
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
  pInfo->isFinal = false;

  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

5926
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
5927 5928
  initResultSizeInfo(&pOperator->resultInfo, 4096);
  SExprSupp* pSup = &pOperator->exprSupp;
5929 5930
  size_t     keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  int32_t    code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  initBasicInfo(&pInfo->binfo, pResBlock);
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
  pInfo->delIndex = 0;
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
  initResultRowInfo(&pInfo->binfo.resultRowInfo);

  pOperator->name = "StreamIntervalOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
5952 5953 5954
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamIntervalOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5955

5
54liuyao 已提交
5956
  initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
  destroyStreamIntervalOperatorInfo(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}