timewindowoperator.c 203.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15
#include "executorimpl.h"
16
#include "filter.h"
X
Xiaoyu Wang 已提交
17
#include "function.h"
5
54liuyao 已提交
18
#include "functionMgt.h"
H
Haojun Liao 已提交
19
#include "tcommon.h"
5
54liuyao 已提交
20
#include "tcompare.h"
L
Liu Jicong 已提交
21
#include "tdatablock.h"
H
Haojun Liao 已提交
22
#include "tfill.h"
23
#include "ttime.h"
24 25 26 27 28 29

typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;

5
54liuyao 已提交
30 31
#define IS_FINAL_OP(op) ((op)->isFinal)

5
54liuyao 已提交
32 33
typedef struct SPullWindowInfo {
  STimeWindow window;
L
Liu Jicong 已提交
34
  uint64_t    groupId;
5
54liuyao 已提交
35 36
} SPullWindowInfo;

37 38
typedef struct SOpenWindowInfo {
  SResultRowPosition pos;
L
Liu Jicong 已提交
39
  uint64_t           groupId;
40 41
} SOpenWindowInfo;

42 43
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);

L
Liu Jicong 已提交
44 45
static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult,
                                              uint64_t groupId);
46 47
static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult);

X
Xiaoyu Wang 已提交
48
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }
49 50 51

static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
52
                                      int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
53 54 55 56 57 58 59 60 61 62 63
                                      SExecTaskInfo* pTaskInfo) {
  SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                  masterscan, tableGroupId, pTaskInfo, true, pAggSup);

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // set time window for current result
  pResultRow->win = (*win);
64

65
  *pResult = pResultRow;
66
  setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
67

68 69 70 71 72 73 74 75 76 77 78 79 80
  return TSDB_CODE_SUCCESS;
}

static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) {
  int64_t* ts = (int64_t*)pColData->pData;
  int32_t  delta = includeEndpoint ? 1 : 0;

  int64_t duration = pWin->ekey - pWin->skey + delta;
  ts[2] = duration;            // set the duration
  ts[3] = pWin->skey;          // window start key
  ts[4] = pWin->ekey + delta;  // window end key
}

81
static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) {
82 83 84
  pRowSup->win.ekey = ts;
  pRowSup->prevTs = ts;
  pRowSup->numOfRows += 1;
85
  pRowSup->groupId = groupId;
86 87
}

dengyihao's avatar
dengyihao 已提交
88 89
static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex,
                                     uint64_t groupId) {
90 91 92
  pRowSup->startRowIndex = rowIndex;
  pRowSup->numOfRows = 0;
  pRowSup->win.skey = tsList[rowIndex];
93
  pRowSup->groupId = groupId;
94 95 96 97
}

static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
                                                   int16_t pos, int16_t order, int64_t* pData) {
98
  int32_t forwardRows = 0;
99 100 101 102

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
103
      forwardRows = end;
104 105

      if (pData[end + pos] == ekey) {
106
        forwardRows += 1;
107 108 109
      }
    }
  } else {
110
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
111
    if (end >= 0) {
112
      forwardRows = end;
113

114
      if (pData[end + pos] == ekey) {
115
        forwardRows += 1;
116 117
      }
    }
X
Xiaoyu Wang 已提交
118 119 120 121 122 123 124 125
    //    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
    //    if (end >= 0) {
    //      forwardRows = pos - end;
    //
    //      if (pData[end] == ekey) {
    //        forwardRows += 1;
    //      }
    //    }
126 127
  }

128 129
  assert(forwardRows >= 0);
  return forwardRows;
130 131
}

5
54liuyao 已提交
132
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY*  keyList = (TSKEY*)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
149 150 151 152 153 154 155 156 157 158 159
      if (key >= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key < keyList[lastPos]) {
        lastPos += 1;
        if (lastPos >= num) {
          return -1;
        } else {
          return lastPos;
        }
      }
160 161 162 163 164 165

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        firstPos = midPos + 1;
166 167
      } else if (key > keyList[midPos]) {
        lastPos = midPos - 1;
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

X
Xiaoyu Wang 已提交
203 204
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
                                 __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num = -1;
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);

  if (order == TSDB_ORDER_ASC) {
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
      if (item != NULL) {
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
226
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
227 228
      }
    } else {
229
      num = pDataBlockInfo->rows - startPos;
230
      if (item != NULL) {
231
        item->lastKey = pDataBlockInfo->window.ekey + step;
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
      }
    }
  }

  assert(num >= 0);
  return num;
}

static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
    return;
  }

  int64_t key = tw->skey, interval = pInterval->interval;
  // convert key to second
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;

  if (pInterval->intervalUnit == 'y') {
    interval *= 12;
  }

  struct tm tm;
  time_t    t = (time_t)key;
  taosLocalTime(&t, &tm);

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
263
  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
264 265 266 267

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
268
  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
269 270 271 272

  tw->ekey -= 1;
}

5
54liuyao 已提交
273 274 275 276
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) {
  getNextTimeWindow(pInterval, pInterval->precision, order, tw);
}

277 278
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs,
                               int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
279
  SqlFunctionCtx* pCtx = pSup->pCtx;
280

281
  int32_t index = 1;
282
  for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
H
Haojun Liao 已提交
283
    if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
284 285 286 287
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

X
Xiaoyu Wang 已提交
288
    SFunctParam*     pParam = &pCtx[k].param[0];
289 290
    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId);

291
    ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey);
292

293
    double v1 = 0, v2 = 0, v = 0;
294
    if (prevRowIndex == -1) {
295
      SGroupKeys* p = taosArrayGet(pPrevValues, index);
296
      GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData);
297
    } else {
298
      GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex));
299 300
    }

301
    GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
302

303
#if 0
304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            //            pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
323 324
#endif

X
Xiaoyu Wang 已提交
325 326 327
    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
    SPoint point = (SPoint){.key = windowKey, .val = &v};
328

X
Xiaoyu Wang 已提交
329
    taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
330

X
Xiaoyu Wang 已提交
331 332 333 334 335 336
    if (type == RESULT_ROW_START_INTERP) {
      pCtx[k].start.key = point.key;
      pCtx[k].start.val = v;
    } else {
      pCtx[k].end.key = point.key;
      pCtx[k].end.val = v;
337
    }
X
Xiaoyu Wang 已提交
338 339 340

    index += 1;
  }
341
#if 0
342
  }
343
#endif
344 345 346 347 348 349 350 351 352 353 354 355 356 357
}

static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

358 359
static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock,
                                              const TSKEY* tsCols, STimeWindow* win, SExprSupp* pSup) {
360
  bool ascQuery = (pInfo->inputOrder == TSDB_ORDER_ASC);
361

362
  TSKEY curTs = tsCols[pos];
363 364

  SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
X
Xiaoyu Wang 已提交
365
  TSKEY       lastTs = *(int64_t*)pTsKey->pData;
366 367 368 369 370

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery ? win->skey : win->ekey;
  if (key == curTs) {
371
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
372 373 374
    return true;
  }

375 376
  // it is the first time window, no need to do interpolation
  if (pTsKey->isNull && pos == 0) {
377
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
378 379
  } else {
    TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]);
380 381
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key,
                              RESULT_ROW_START_INTERP, pSup);
382 383 384 385 386
  }

  return true;
}

387 388 389
static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex,
                                            SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey,
                                            STimeWindow* win) {
390
  int32_t order = pInfo->inputOrder;
391 392

  TSKEY actualEndKey = tsCols[endRowIndex];
393
  TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey;
394 395

  // not ended in current data block, do not invoke interpolation
396
  if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) {
397
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
398 399 400
    return false;
  }

401
  // there is actual end point of current time window, no interpolation needs
402
  if (key == actualEndKey) {
403
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
404 405 406
    return true;
  }

407
  int32_t nextRowIndex = endRowIndex + 1;
408 409 410
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
411 412
  doTimeWindowInterpolation(pInfo->pPrevValues, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key,
                            RESULT_ROW_END_INTERP, pSup);
413 414 415
  return true;
}

416 417
bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd) {
  if (pInterval->interval != pInterval->sliding && (pWin->ekey < calStart || pWin->skey > calEnd)) {
5
54liuyao 已提交
418 419 420 421 422
    return false;
  }
  return true;
}

423 424 425 426
bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo) {
  return inCalSlidingWindow(pInterval, pWin, pBlockInfo->calWin.skey, pBlockInfo->calWin.ekey);
}

427
static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
5
54liuyao 已提交
428
                                      TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
X
Xiaoyu Wang 已提交
429
  bool ascQuery = (order == TSDB_ORDER_ASC);
430 431 432 433 434 435 436 437 438 439

  int32_t precision = pInterval->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
    return -1;
  }

5
54liuyao 已提交
440 441 442 443
  if (!inSlidingWindow(pInterval, pNext, pDataBlockInfo) && order == TSDB_ORDER_ASC) {
    return -1;
  }

444
  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
445 446 447 448
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
449
    startPos = prevPosition + 1;
450
  } else {
451
    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
452 453
      startPos = 0;
    } else {
454
      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497
    }
  }

  /* interp query with fill should not skip time window */
  //  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
  //    return startPos;
  //  }

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
    if (ascQuery) {
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
      }
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
      }
    }
  }

  return startPos;
}

498 499
static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
  ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp == true;
  }
}

static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp = true;
  }
}

516 517
static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult,
                                        STimeWindow* win, int32_t startPos, int32_t forwardRows, SExprSupp* pSup) {
518
  if (!pInfo->timeWindowInterpo) {
519 520 521
    return;
  }

522
  ASSERT(pBlock != NULL);
523 524 525 526 527
  if (pBlock->pDataBlock == NULL) {
    //    tscError("pBlock->pDataBlock == NULL");
    return;
  }

528
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
529 530

  TSKEY* tsCols = (TSKEY*)(pColInfo->pData);
531
  bool   done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
532
  if (!done) {  // it is not interpolated, now start to generated the interpolated value
533
    bool interp = setTimeWindowInterpolationStartTs(pInfo, startPos, pBlock, tsCols, win, pSup);
534 535 536 537
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
538
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
539 540 541 542 543 544 545 546
  }

  // point interpolation does not require the end key time window interpolation.
  //  if (pointInterpQuery) {
  //    return;
  //  }

  // interpolation query does not generate the time window end interpolation
547
  done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
548
  if (!done) {
549
    int32_t endRowIndex = startPos + forwardRows - 1;
550

551
    TSKEY endKey = (pInfo->inputOrder == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
552
    bool  interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
553 554 555 556
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
557
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
558 559 560
  }
}

561 562
static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) {
  if (pBlock->pDataBlock == NULL) {
563 564 565
    return;
  }

566 567 568 569 570 571 572
  size_t num = taosArrayGetSize(pPrevKeys);
  for (int32_t k = 0; k < num; ++k) {
    SColumn* pc = taosArrayGet(pCols, k);

    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId);

    SGroupKeys* pkey = taosArrayGet(pPrevKeys, k);
X
Xiaoyu Wang 已提交
573
    for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) {
574 575 576 577 578 579 580 581 582 583 584 585 586 587
      if (colDataIsNull_s(pColInfo, i)) {
        continue;
      }

      char* val = colDataGetData(pColInfo, i);
      if (IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, varDataTLen(val));
        ASSERT(varDataTLen(val) <= pkey->bytes);
      } else {
        memcpy(pkey->pData, val, pkey->bytes);
      }

      break;
    }
588 589 590
  }
}

591 592 593 594
static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo,
                                       SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) {
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

595
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
596
  SExprSupp*                pSup = &pOperatorInfo->exprSupp;
597

L
Liu Jicong 已提交
598 599
  int32_t startPos = 0;
  int32_t numOfOutput = pSup->numOfExprs;
600

601
  SResultRow* pResult = NULL;
602

603
  while (1) {
L
Liu Jicong 已提交
604 605 606
    SListNode*          pn = tdListGetHead(pResultRowInfo->openWindow);
    SOpenWindowInfo*    pOpenWin = (SOpenWindowInfo*)pn->data;
    uint64_t            groupId = pOpenWin->groupId;
607
    SResultRowPosition* p1 = &pOpenWin->pos;
608 609 610
    if (p->pageId == p1->pageId && p->offset == p1->offset) {
      break;
    }
611

612
    SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1, false);
613
    ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
614

615
    if (pr->closed) {
X
Xiaoyu Wang 已提交
616 617
      ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
             isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
618 619
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
620 621
      continue;
    }
622

623
    STimeWindow w = pr->win;
624 625
    int32_t     ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx,
                                             numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
626
    if (ret != TSDB_CODE_SUCCESS) {
627
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
628 629 630 631
    }

    ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP));

X
Xiaoyu Wang 已提交
632 633
    SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
    int64_t     prevTs = *(int64_t*)pTsKey->pData;
634 635 636 637
    if (groupId == pBlock->info.groupId) {
      doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey,
                                RESULT_ROW_END_INTERP, pSup);
    }
638 639

    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
640
    setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
641

642
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, true);
H
Haojun Liao 已提交
643 644
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, pBlock->info.rows,
                     numOfExprs);
645 646 647

    if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
      closeResultRow(pr);
648 649
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
X
Xiaoyu Wang 已提交
650
    } else {  // the remains are can not be closed yet.
651
      break;
652
    }
653
  }
654
}
655

5
54liuyao 已提交
656
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
5
54liuyao 已提交
657
  if (!pBlock || pBlock->info.rows == 0) {
5
54liuyao 已提交
658
    qDebug("===stream===printDataBlock: Block is Null or Empty");
5
54liuyao 已提交
659 660
    return;
  }
L
Liu Jicong 已提交
661
  char* pBuf = NULL;
5
54liuyao 已提交
662 663
  qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
  taosMemoryFree(pBuf);
5
54liuyao 已提交
664 665
}

5
54liuyao 已提交
666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721
typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);

int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;
      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;
      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;
      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

      if (comparefn(pKey, keyList, lastPos) > 0) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

5
54liuyao 已提交
722
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
723

X
Xiaoyu Wang 已提交
724 725 726
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;
5
54liuyao 已提交
727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (key >= getValuefn(keyList, lastPos)) return lastPos;
      if (key == getValuefn(keyList, firstPos)) return firstPos;
      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (key <= getValuefn(keyList, firstPos)) return firstPos;
      if (key == getValuefn(keyList, lastPos)) return lastPos;

      if (key > getValuefn(keyList, lastPos)) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
773 774
  }

5
54liuyao 已提交
775 776 777
  return midPos;
}

5
54liuyao 已提交
778
int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {
L
Liu Jicong 已提交
779
  SArray*          res = (SArray*)data;
5
54liuyao 已提交
780
  SPullWindowInfo* pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
781
  SPullWindowInfo* pData = (SPullWindowInfo*)pKey;
5
54liuyao 已提交
782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812
  if (pData->window.skey == pos->window.skey) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->window.skey > pos->window.skey) {
    return 1;
  }
  return -1;
}

static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {
  int32_t size = taosArrayGetSize(pPullWins);
  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);
  if (index == -1) {
    index = 0;
  } else {
    if (comparePullWinKey(pPullInfo, pPullWins, index) > 0) {
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
813 814 815
static int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) {
  winInfo.sessionWin.win.ekey = winInfo.sessionWin.win.skey;
  return tSimpleHashPut(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));
5
54liuyao 已提交
816 817
}

5
54liuyao 已提交
818 819 820 821 822 823 824 825
static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SHashObj* pUpdatedMap) {
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
H
Haojun Liao 已提交
826 827
  SWinKey key = {.ts = ts, .groupId = groupId};
  if (taosHashPut(pUpdatedMap, &key, sizeof(SWinKey), &newPos, sizeof(void*)) != TSDB_CODE_SUCCESS) {
5
54liuyao 已提交
828 829 830
    taosMemoryFree(newPos);
  }
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
831 832
}

5
54liuyao 已提交
833 834 835 836
static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SHashObj* pUpdatedMap) {
  return saveWinResult(ts, -1, -1, groupId, pUpdatedMap);
}

5
54liuyao 已提交
837
static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) {
5
54liuyao 已提交
838 839
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
H
Haojun Liao 已提交
840
    SWinKey* pW = taosArrayGet(pWins, i);
5
54liuyao 已提交
841 842 843 844 845 846
    void*    tmp = taosHashGet(pUpdatedMap, pW, sizeof(SWinKey));
    if (tmp) {
      void* value = *(void**)tmp;
      taosMemoryFree(value);
      taosHashRemove(pUpdatedMap, pW, sizeof(SWinKey));
    }
5
54liuyao 已提交
847 848 849
  }
}

5
54liuyao 已提交
850 851
int32_t compareWinRes(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
852
  SWinKey*    pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
853
  SResKeyPos* pData = (SResKeyPos*)pKey;
5
54liuyao 已提交
854 855 856 857 858 859 860 861 862 863 864 865 866
  if (*(int64_t*)pData->key == pos->ts) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (*(int64_t*)pData->key > pos->ts) {
    return 1;
  }
  return -1;
}

5
54liuyao 已提交
867
static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) {
5
54liuyao 已提交
868 869
  taosArraySort(pDelWins, winKeyCmprImpl);
  taosArrayRemoveDuplicate(pDelWins, winKeyCmprImpl, NULL);
L
Liu Jicong 已提交
870 871
  int32_t delSize = taosArrayGetSize(pDelWins);
  if (taosHashGetSize(pUpdatedMap) == 0 || delSize == 0) {
5
54liuyao 已提交
872
    return;
dengyihao's avatar
dengyihao 已提交
873
  }
L
Liu Jicong 已提交
874
  void* pIte = NULL;
5
54liuyao 已提交
875
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
876
    SResKeyPos* pResKey = *(SResKeyPos**)pIte;
5
54liuyao 已提交
877 878
    int32_t     index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes);
    if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) {
879
      taosArrayRemove(pDelWins, index);
880
      delSize = taosArrayGetSize(pDelWins);
881 882 883 884
    }
  }
}

5
54liuyao 已提交
885 886 887
bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) {
  ASSERT(pTwSup->maxTs == INT64_MIN || pTwSup->maxTs > 0);
  return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark;
5
54liuyao 已提交
888 889
}

5
54liuyao 已提交
890 891 892 893 894
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); }

bool needDeleteWindowBuf(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) {
  return pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark;
}
5
54liuyao 已提交
895

5
54liuyao 已提交
896
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
897
                            int32_t scanFlag) {
898
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
899

900
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
901
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
902

X
Xiaoyu Wang 已提交
903
  int32_t     startPos = 0;
904
  int32_t     numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
905 906
  int64_t*    tsCols = extractTsCol(pBlock, pInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
907
  bool        ascScan = (pInfo->inputOrder == TSDB_ORDER_ASC);
X
Xiaoyu Wang 已提交
908 909
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;
910

911 912
  STimeWindow win =
      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder);
913 914
  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
915 916
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
917
  }
X
Xiaoyu Wang 已提交
918 919
  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
920
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
921
  ASSERT(forwardRows > 0);
922 923

  // prev time window not interpolation yet.
924
  if (pInfo->timeWindowInterpo) {
925
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
926
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);
927 928

    // restore current time window
929 930
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
931
    if (ret != TSDB_CODE_SUCCESS) {
932
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
933 934
    }

935
    // window start key interpolation
936
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
937
  }
938

939 940
  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
941
                   numOfOutput);
942 943

  doCloseWindow(pResultRowInfo, pInfo, pResult);
944 945 946

  STimeWindow nextWin = win;
  while (1) {
947
    int32_t prevEndPos = forwardRows - 1 + startPos;
948
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->inputOrder);
949 950 951 952
    if (startPos < 0) {
      break;
    }
    // null data, failed to allocate more memory buffer
X
Xiaoyu Wang 已提交
953
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
954
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
955
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
956
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
957 958
    }

X
Xiaoyu Wang 已提交
959
    ekey = ascScan ? nextWin.ekey : nextWin.skey;
960
    forwardRows =
961
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
962
    // window start(end) key interpolation
963
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
L
Liu Jicong 已提交
964
    // TODO: add to open window? how to close the open windows after input blocks exhausted?
S
shenglian zhou 已提交
965
#if 0
966 967 968 969
    if ((ascScan && ekey <= pBlock->info.window.ekey) ||
        (!ascScan && ekey >= pBlock->info.window.skey)) {
      // window start(end) key interpolation
      doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
970
    } else if (pInfo->timeWindowInterpo) {
971 972
      addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
    }
S
shenglian zhou 已提交
973
#endif
974
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
H
Haojun Liao 已提交
975 976
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
977
    doCloseWindow(pResultRowInfo, pInfo, pResult);
978 979 980
  }

  if (pInfo->timeWindowInterpo) {
981
    saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
982
  }
983 984 985 986 987 988
}

void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
  // current result is done in computing final results.
  if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
    closeResultRow(pResult);
989
    SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
D
dapan1121 已提交
990
    taosMemoryFree(pNode);
991 992 993
  }
}

994 995 996 997 998
SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, uint64_t groupId) {
  SOpenWindowInfo openWin = {0};
  openWin.pos.pageId = pResult->pageId;
  openWin.pos.offset = pResult->offset;
  openWin.groupId = groupId;
L
Liu Jicong 已提交
999
  SListNode* pn = tdListGetTail(pResultRowInfo->openWindow);
1000
  if (pn == NULL) {
1001 1002
    tdListAppend(pResultRowInfo->openWindow, &openWin);
    return openWin.pos;
1003 1004
  }

L
Liu Jicong 已提交
1005
  SOpenWindowInfo* px = (SOpenWindowInfo*)pn->data;
1006 1007
  if (px->pos.pageId != openWin.pos.pageId || px->pos.offset != openWin.pos.offset || px->groupId != openWin.groupId) {
    tdListAppend(pResultRowInfo->openWindow, &openWin);
1008 1009
  }

1010
  return openWin.pos;
1011 1012 1013 1014
}

int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
  TSKEY* tsCols = NULL;
1015

1016 1017 1018 1019
  if (pBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;

1020 1021 1022 1023 1024 1025
    // no data in primary ts
    if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) {
      return NULL;
    }

    if (tsCols[0] != 0 && (pBlock->info.window.skey == 0 && pBlock->info.window.ekey == 0)) {
1026 1027 1028 1029 1030
      blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
    }
  }

  return tsCols;
1031 1032 1033 1034 1035 1036 1037
}

static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
  }

L
Liu Jicong 已提交
1038
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1039
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
1040
  SExprSupp*                pSup = &pOperator->exprSupp;
1041

1042 1043
  int32_t scanFlag = MAIN_SCAN;

X
Xiaoyu Wang 已提交
1044
  int64_t        st = taosGetTimestampUs();
1045 1046 1047
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1048
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1049 1050 1051 1052
    if (pBlock == NULL) {
      break;
    }

1053
    getTableScanInfo(pOperator, &pInfo->inputOrder, &scanFlag);
1054

1055
    if (pInfo->scalarSupp.pExprInfo != NULL) {
L
Liu Jicong 已提交
1056 1057
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
1058 1059
    }

1060
    // the pDataBlock are always the same one, no need to call this again
1061
    setInputDataBlock(pSup, pBlock, pInfo->inputOrder, scanFlag, true);
1062 1063
    blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);

1064
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag);
1065 1066
  }

1067
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->resultTsOrder);
1068
  OPTR_SET_OPENED(pOperator);
1069 1070

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1071 1072 1073
  return TSDB_CODE_SUCCESS;
}

1074 1075 1076 1077 1078
static bool compareVal(const char* v, const SStateKeys* pKey) {
  if (IS_VAR_DATA_TYPE(pKey->type)) {
    if (varDataLen(v) != varDataLen(pKey->pData)) {
      return false;
    } else {
D
dapan1121 已提交
1079
      return memcmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0;
1080 1081 1082 1083 1084 1085
    }
  } else {
    return memcmp(pKey->pData, v, pKey->bytes) == 0;
  }
}

1086
static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
L
Liu Jicong 已提交
1087
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1088
  SExprSupp*     pSup = &pOperator->exprSupp;
1089

1090
  SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
1091 1092 1093
  int64_t          gid = pBlock->info.groupId;

  bool    masterScan = true;
1094
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1095 1096
  int16_t bytes = pStateColInfoData->info.bytes;

1097
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1098 1099 1100 1101 1102
  TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

1103
  struct SColumnDataAgg* pAgg = NULL;
1104
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
X
Xiaoyu Wang 已提交
1105
    pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
1106
    if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
1107 1108 1109 1110 1111
      continue;
    }

    char* val = colDataGetData(pStateColInfoData, j);

1112
    if (gid != pRowSup->groupId || !pInfo->hasKey) {
1113 1114 1115 1116 1117 1118 1119
      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }

1120 1121
      pInfo->hasKey = true;

1122 1123
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1124
    } else if (compareVal(val, &pInfo->stateKey)) {
1125
      doKeepTuple(pRowSup, tsList[j], gid);
1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // a new state window started
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1136 1137
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1138
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1139
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1140 1141 1142
      }

      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1143 1144
      doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                       pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1145 1146

      // here we start a new session window
1147 1148
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1149 1150 1151 1152 1153 1154 1155

      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }
1156 1157 1158 1159 1160
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1161 1162
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1163
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1164
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1165 1166 1167
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
H
Haojun Liao 已提交
1168 1169
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
                   pBlock->info.rows, numOfOutput);
1170 1171
}

H
Hongze Cheng 已提交
1172
static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) {
1173 1174
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
1175 1176 1177
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
1178
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1179

1180 1181 1182
  SExprSupp* pSup = &pOperator->exprSupp;
  int32_t    order = TSDB_ORDER_ASC;
  int64_t    st = taosGetTimestampUs();
1183 1184 1185

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  while (1) {
1186
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1187 1188 1189 1190
    if (pBlock == NULL) {
      break;
    }

1191
    setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
1192 1193
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1194 1195 1196 1197 1198 1199 1200 1201 1202
    // there is an scalar expression that needs to be calculated right before apply the group aggregation.
    if (pInfo->scalarSup.pExprInfo != NULL) {
      pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
                                              pInfo->scalarSup.numOfExprs, NULL);
      if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
        T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
      }
    }

1203 1204 1205
    doStateWindowAggImpl(pOperator, pInfo, pBlock);
  }

X
Xiaoyu Wang 已提交
1206
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1207
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1208 1209
  pOperator->status = OP_RES_TO_RETURN;

1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223
  return TSDB_CODE_SUCCESS;
}

static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
  SOptrBasicInfo*           pBInfo = &pInfo->binfo;

  pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
  if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
H
Haojun Liao 已提交
1224
    setOperatorCompleted(pOperator);
1225 1226 1227
    return NULL;
  }

1228
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1229
  while (1) {
1230
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
H
Haojun Liao 已提交
1231
    doFilter(pBInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
1232

1233
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1234
    if (!hasRemain) {
H
Haojun Liao 已提交
1235
      setOperatorCompleted(pOperator);
1236 1237
      break;
    }
1238

1239 1240 1241 1242
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
1243

1244
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1245
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1246 1247
}

1248
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
1249
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
1250
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1251 1252 1253 1254 1255 1256 1257

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pBlock = pInfo->binfo.pRes;

1258
  ASSERT(pInfo->execModel == OPTR_EXEC_MODEL_BATCH);
1259

1260 1261 1262 1263
  pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
  if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
    return NULL;
  }
1264

1265 1266 1267 1268
  blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
  while (1) {
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
    doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL);
1269

1270 1271
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
    if (!hasRemain) {
H
Haojun Liao 已提交
1272
      setOperatorCompleted(pOperator);
1273
      break;
1274 1275
    }

1276 1277 1278
    if (pBlock->info.rows > 0) {
      break;
    }
1279
  }
1280 1281 1282 1283 1284

  size_t rows = pBlock->info.rows;
  pOperator->resultInfo.totalRows += rows;

  return (rows == 0) ? NULL : pBlock;
1285 1286
}

5
54liuyao 已提交
1287
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
L
Liu Jicong 已提交
1288
  for (int i = 0; i < num; i++) {
5
54liuyao 已提交
1289 1290
    if (type == STREAM_INVERT) {
      fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
L
Liu Jicong 已提交
1291
    } else if (type == STREAM_NORMAL) {
5
54liuyao 已提交
1292 1293 1294 1295
      fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
    }
  }
}
5
54liuyao 已提交
1296

5
54liuyao 已提交
1297
static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) {
1298
  SResultRow*     pResult = getResultRowByPos(pResultBuf, p1, false);
1299
  SqlFunctionCtx* pCtx = pSup->pCtx;
5
54liuyao 已提交
1300
  for (int32_t i = 0; i < numOfOutput; ++i) {
1301
    pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
1302 1303 1304 1305 1306 1307 1308 1309 1310
    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
      continue;
    }
    pResInfo->initialized = false;
    if (pCtx[i].functionId != -1) {
      pCtx[i].fpSet.init(&pCtx[i], pResInfo);
    }
  }
5
54liuyao 已提交
1311 1312 1313
  SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pResultBuf, bufPage);
5
54liuyao 已提交
1314 1315
}

1316
static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) {
5
54liuyao 已提交
1317 1318 1319
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
  SWinKey                      key = {.ts = ts, .groupId = groupId};
  tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));
1320
  streamStateDel(pInfo->pState, &key);
5
54liuyao 已提交
1321 1322 1323
  return true;
}

1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins,
                            SHashObj* pUpdatedMap) {
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
  SColumnInfoData*             pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  TSKEY*                       startTsCols = (TSKEY*)pStartTsCol->pData;
  SColumnInfoData*             pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*                       endTsCols = (TSKEY*)pEndTsCol->pData;
  SColumnInfoData*             pCalStTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
  TSKEY*                       calStTsCols = (TSKEY*)pCalStTsCol->pData;
  SColumnInfoData*             pCalEnTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
  TSKEY*                       calEnTsCols = (TSKEY*)pCalEnTsCol->pData;
  SColumnInfoData*             pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*                    pGpDatas = (uint64_t*)pGpCol->pData;
5
54liuyao 已提交
1337
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
H
Haojun Liao 已提交
1338
    SResultRowInfo dumyInfo = {0};
5
54liuyao 已提交
1339
    dumyInfo.cur.pageId = -1;
H
Haojun Liao 已提交
1340

1341 1342 1343 1344 1345 1346 1347 1348
    STimeWindow win = {0};
    if (IS_FINAL_OP(pInfo)) {
      win.skey = startTsCols[i];
      win.ekey = endTsCols[i];
    } else {
      win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
    }

5
54liuyao 已提交
1349
    do {
1350 1351 1352 1353
      if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i])) {
        getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
        continue;
      }
5
54liuyao 已提交
1354
      uint64_t winGpId = pGpDatas[i];
1355
      bool     res = doDeleteWindow(pOperator, win.skey, winGpId);
5
54liuyao 已提交
1356 1357 1358 1359 1360
      SWinKey  winRes = {.ts = win.skey, .groupId = winGpId};
      if (pUpWins && res) {
        taosArrayPush(pUpWins, &winRes);
      }
      if (pUpdatedMap) {
5
54liuyao 已提交
1361 1362 1363 1364 1365 1366
        void* tmp = taosHashGet(pUpdatedMap, &winRes, sizeof(SWinKey));
        if (tmp) {
          void* value = *(void**)tmp;
          taosMemoryFree(value);
          taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
        }
5
54liuyao 已提交
1367 1368
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
5
54liuyao 已提交
1369
    } while (win.ekey <= endTsCols[i]);
5
54liuyao 已提交
1370 1371 1372
  }
}

1373 1374 1375 1376 1377 1378
static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) {
  void*   pIte = NULL;
  size_t  keyLen = 0;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    void*    key = tSimpleHashGetKey(pIte, &keyLen);
5
54liuyao 已提交
1379 1380
    uint64_t groupId = *(uint64_t*)key;
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
1381
    TSKEY               ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1382
    SResultRowPosition* pPos = (SResultRowPosition*)pIte;
5
54liuyao 已提交
1383
    int32_t             code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
5
54liuyao 已提交
1384 1385 1386 1387 1388 1389 1390
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }
  return TSDB_CODE_SUCCESS;
}

1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407
int32_t compareWinKey(void* pKey, void* data, int32_t index) {
  SArray*  res = (SArray*)data;
  SWinKey* pos = taosArrayGet(res, index);
  SWinKey* pData = (SWinKey*)pKey;
  if (pData->ts == pos->ts) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->ts > pos->ts) {
    return 1;
  }
  return -1;
}

5
54liuyao 已提交
1408
static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval,
1409 1410
                                         SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pDelWins,
                                         SOperatorInfo* pOperator) {
5
54liuyao 已提交
1411
  qDebug("===stream===close interval window");
1412 1413 1414 1415
  void*                        pIte = NULL;
  size_t                       keyLen = 0;
  int32_t                      iter = 0;
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
1416
  int32_t                      delSize = taosArrayGetSize(pDelWins);
5
54liuyao 已提交
1417
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
    void*    key = tSimpleHashGetKey(pIte, &keyLen);
    SWinKey* pWinKey = (SWinKey*)key;
    if (delSize > 0) {
      int32_t index = binarySearchCom(pDelWins, delSize, pWinKey, TSDB_ORDER_DESC, compareWinKey);
      if (index >= 0 && 0 == compareWinKey(pWinKey, pDelWins, index)) {
        taosArrayRemove(pDelWins, index);
        delSize = taosArrayGetSize(pDelWins);
      }
    }

5
54liuyao 已提交
1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452
    void*       chIds = taosHashGet(pPullDataMap, pWinKey, sizeof(SWinKey));
    STimeWindow win = {
        .skey = pWinKey->ts,
        .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1,
    };
    if (isCloseWindow(&win, pTwSup)) {
      if (chIds && pPullDataMap) {
        SArray* chAy = *(SArray**)chIds;
        int32_t size = taosArrayGetSize(chAy);
        qDebug("===stream===window %" PRId64 " wait child size:%d", pWinKey->ts, size);
        for (int32_t i = 0; i < size; i++) {
          qDebug("===stream===window %" PRId64 " wait child id:%d", pWinKey->ts, *(int32_t*)taosArrayGet(chAy, i));
        }
        continue;
      } else if (pPullDataMap) {
        qDebug("===stream===close window %" PRId64, pWinKey->ts);
      }

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
        int32_t code = saveWinResultInfo(pWinKey->ts, pWinKey->groupId, closeWins);
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
      }
      tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter);
1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497
    }
  }
  return TSDB_CODE_SUCCESS;
}

STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
  return w;
}

static void deleteIntervalDiscBuf(SStreamState* pState, SHashObj* pPullDataMap, TSKEY mark, SInterval* pInterval,
                                  SWinKey* key) {
  STimeWindow tw = getFinalTimeWindow(key->ts, pInterval);
  SWinKey     next = {0};
  while (tw.ekey < mark) {
    SStreamStateCur* pCur = streamStateSeekKeyNext(pState, key);
    int32_t          code = streamStateGetKVByCur(pCur, &next, NULL, 0);
    streamStateFreeCur(pCur);

    void* chIds = taosHashGet(pPullDataMap, key, sizeof(SWinKey));
    if (chIds && pPullDataMap) {
      SArray* chAy = *(SArray**)chIds;
      int32_t size = taosArrayGetSize(chAy);
      qDebug("===stream===window %" PRId64 " wait child size:%d", key->ts, size);
      for (int32_t i = 0; i < size; i++) {
        qDebug("===stream===window %" PRId64 " wait child id:%d", key->ts, *(int32_t*)taosArrayGet(chAy, i));
      }
      break;
    }
    qDebug("===stream===delete window %" PRId64, key->ts);
    int32_t codeDel = streamStateDel(pState, key);
    if (codeDel != TSDB_CODE_SUCCESS) {
      code = streamStateGetFirst(pState, key);
      if (code != TSDB_CODE_SUCCESS) {
        qDebug("===stream===stream state first key: empty-empty");
        return;
      }
      continue;
    }
    if (code == TSDB_CODE_SUCCESS) {
      *key = next;
      tw = getFinalTimeWindow(key->ts, pInterval);
    }
  }
5
54liuyao 已提交
1498

5
54liuyao 已提交
1499 1500
  // for debug
  if (qDebugFlag & DEBUG_DEBUG && mark > 0) {
1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
    SStreamStateCur* pCur = streamStateGetCur(pState, key);
    int32_t          code = streamStateCurPrev(pState, pCur);
    if (code == TSDB_CODE_SUCCESS) {
      SWinKey tmpKey = {0};
      code = streamStateGetKVByCur(pCur, &tmpKey, NULL, 0);
      if (code == TSDB_CODE_SUCCESS) {
        STimeWindow tw = getFinalTimeWindow(tmpKey.ts, pInterval);
        qDebug("===stream===error stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey,
               tw.ekey, tmpKey.groupId, mark);
      } else {
        STimeWindow tw = getFinalTimeWindow(key->ts, pInterval);
        qDebug("===stream===stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey, tw.ekey,
               key->groupId, mark);
5
54liuyao 已提交
1514
      }
1515 1516 1517 1518
    } else {
      STimeWindow tw = getFinalTimeWindow(key->ts, pInterval);
      qDebug("===stream===stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey, tw.ekey,
             key->groupId, mark);
5
54liuyao 已提交
1519
    }
1520
    streamStateFreeCur(pCur);
5
54liuyao 已提交
1521 1522 1523
  }
}

1524
static void closeChildIntervalWindow(SOperatorInfo* pOperator, SArray* pChildren, TSKEY maxTs) {
5
54liuyao 已提交
1525 1526
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
1527 1528
    SOperatorInfo*               pChildOp = taosArrayGetP(pChildren, i);
    SStreamIntervalOperatorInfo* pChInfo = pChildOp->info;
5
54liuyao 已提交
1529 1530
    ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
1531
    closeStreamIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL,
1532
                              NULL, pOperator);
1533 1534 1535
  }
}

1536 1537
static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWins, int32_t* index,
                                SSDataBlock* pBlock) {
1538 1539 1540 1541 1542 1543 1544 1545
  blockDataCleanup(pBlock);
  int32_t size = taosArrayGetSize(pWins);
  if (*index == size) {
    *index = 0;
    taosArrayClear(pWins);
    return;
  }
  blockDataEnsureCapacity(pBlock, size - *index);
1546
  uint64_t uid = 0;
1547
  for (int32_t i = *index; i < size; i++) {
H
Haojun Liao 已提交
1548
    SWinKey* pWin = taosArrayGet(pWins, i);
1549 1550
    void*    tbname = NULL;
    streamStateGetParName(pInfo->pState, pWin->groupId, &tbname);
1551 1552 1553 1554 1555 1556 1557
    if (tbname == NULL) {
      appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL);
    } else {
      char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
      STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
      appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName);
    }
1558
    tdbFree(tbname);
1559
    (*index)++;
5
54liuyao 已提交
1560 1561 1562
  }
}

1563
static void destroyStateWindowOperatorInfo(void* param) {
1564
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
1565
  cleanupBasicInfo(&pInfo->binfo);
1566
  taosMemoryFreeClear(pInfo->stateKey.pData);
1567
  cleanupExprSupp(&pInfo->scalarSup);
D
dapan1121 已提交
1568 1569 1570
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  cleanupAggSup(&pInfo->aggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
1571

D
dapan1121 已提交
1572
  taosMemoryFreeClear(param);
1573 1574
}

H
Haojun Liao 已提交
1575
static void freeItem(void* param) {
L
Liu Jicong 已提交
1576
  SGroupKeys* pKey = (SGroupKeys*)param;
H
Haojun Liao 已提交
1577 1578 1579
  taosMemoryFree(pKey->pData);
}

1580
void destroyIntervalOperatorInfo(void* param) {
1581
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
1582
  cleanupBasicInfo(&pInfo->binfo);
1583
  cleanupAggSup(&pInfo->aggSup);
1584 1585 1586 1587
  cleanupExprSupp(&pInfo->scalarSupp);

  tdListFree(pInfo->binfo.resultRowInfo.openWindow);

H
Haojun Liao 已提交
1588 1589 1590 1591
  pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols);
  taosArrayDestroyEx(pInfo->pPrevValues, freeItem);

  pInfo->pPrevValues = NULL;
1592

H
Haojun Liao 已提交
1593 1594
  cleanupGroupResInfo(&pInfo->groupResInfo);
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
D
dapan1121 已提交
1595
  taosMemoryFreeClear(param);
1596 1597
}

1598
void destroyStreamFinalIntervalOperatorInfo(void* param) {
1599
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;
1600
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
1601
  cleanupAggSup(&pInfo->aggSup);
L
Liu Jicong 已提交
1602
  // it should be empty.
5
54liuyao 已提交
1603 1604 1605
  taosHashCleanup(pInfo->pPullDataMap);
  taosArrayDestroy(pInfo->pPullWins);
  blockDataDestroy(pInfo->pPullDataRes);
L
Liu Jicong 已提交
1606 1607
  taosArrayDestroy(pInfo->pDelWins);
  blockDataDestroy(pInfo->pDelRes);
1608
  taosMemoryFreeClear(pInfo->pState);
5
54liuyao 已提交
1609

1610 1611 1612 1613
  if (pInfo->pChildren) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
5
54liuyao 已提交
1614
      destroyOperatorInfo(pChildOp);
1615
    }
L
Liu Jicong 已提交
1616
    taosArrayDestroy(pInfo->pChildren);
1617
  }
1618
  nodesDestroyNode((SNode*)pInfo->pPhyNode);
5
54liuyao 已提交
1619
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
1620
  cleanupGroupResInfo(&pInfo->groupResInfo);
5
54liuyao 已提交
1621
  cleanupExprSupp(&pInfo->scalarSupp);
1622

D
dapan1121 已提交
1623
  taosMemoryFreeClear(param);
5
54liuyao 已提交
1624 1625
}

1626
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
5
54liuyao 已提交
1627
  for (int32_t i = 0; i < numOfCols; i++) {
5
54liuyao 已提交
1628
    if (fmIsUserDefinedFunc(pFCtx[i].functionId) || !fmIsInvertible(pFCtx[i].functionId)) {
5
54liuyao 已提交
1629 1630 1631 1632 1633 1634
      return false;
    }
  }
  return true;
}

1635
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
1636 1637
  // the primary timestamp column
  bool needed = false;
1638 1639
  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
1640

X
Xiaoyu Wang 已提交
1641
  {  // ts column
1642 1643
    SColumn c = {0};
    c.colId = 1;
1644
    c.slotId = pInfo->primaryTsIndex;
1645 1646
    c.type = TSDB_DATA_TYPE_TIMESTAMP;
    c.bytes = sizeof(int64_t);
1647
    taosArrayPush(pInfo->pInterpCols, &c);
1648 1649

    SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1650 1651 1652 1653
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = true;  // to denote no value is assigned yet
    key.pData = taosMemoryCalloc(1, c.bytes);
1654
    taosArrayPush(pInfo->pPrevValues, &key);
1655 1656
  }

X
Xiaoyu Wang 已提交
1657
  for (int32_t i = 0; i < numOfCols; ++i) {
1658 1659
    SExprInfo* pExpr = pCtx[i].pExpr;

H
Haojun Liao 已提交
1660
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1661 1662 1663
      SFunctParam* pParam = &pExpr->base.pParam[0];

      SColumn c = *pParam->pCol;
1664
      taosArrayPush(pInfo->pInterpCols, &c);
1665 1666 1667
      needed = true;

      SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1668 1669
      key.bytes = c.bytes;
      key.type = c.type;
1670
      key.isNull = false;
X
Xiaoyu Wang 已提交
1671
      key.pData = taosMemoryCalloc(1, c.bytes);
1672
      taosArrayPush(pInfo->pPrevValues, &key);
1673 1674 1675 1676 1677 1678
    }
  }

  return needed;
}

L
Liu Jicong 已提交
1679
void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval,
5
54liuyao 已提交
1680
                            STimeWindowAggSupp* pTwSup) {
1681
  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
5
54liuyao 已提交
1682
    initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, pTwSup);
1683 1684
    return;
  }
5
54liuyao 已提交
1685
  SStreamScanInfo* pScanInfo = downstream->info;
1686 1687
  pScanInfo->windowSup.parentType = type;
  pScanInfo->windowSup.pIntervalAggSup = pSup;
5
54liuyao 已提交
1688 1689 1690
  if (!pScanInfo->pUpdateInfo) {
    pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, pTwSup->waterMark);
  }
1691
  pScanInfo->interval = *pInterval;
5
54liuyao 已提交
1692
  pScanInfo->twAggSup = *pTwSup;
5
54liuyao 已提交
1693 1694
}

H
Haojun Liao 已提交
1695 1696 1697 1698 1699 1700
void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
  for (int32_t i = 0; i < numOfExpr; i++) {
    pCtx[i].isStream = true;
  }
}

H
Haojun Liao 已提交
1701
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode,
L
Liu Jicong 已提交
1702
                                          SExecTaskInfo* pTaskInfo, bool isStream) {
1703
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1704
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1705 1706 1707 1708
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

H
Haojun Liao 已提交
1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->window.node.pOutputDataBlockDesc);
  initBasicInfo(&pInfo->binfo, pResBlock);

  SExprSupp* pSup = &pOperator->exprSupp;
  pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId;

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(&pOperator->resultInfo, 4096);

  int32_t    num = 0;
  SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
  int32_t    code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  SInterval interval = {.interval = pPhyNode->interval,
1726 1727 1728 1729 1730
                        .sliding = pPhyNode->sliding,
                        .intervalUnit = pPhyNode->intervalUnit,
                        .slidingUnit = pPhyNode->slidingUnit,
                        .offset = pPhyNode->offset,
                        .precision = ((SColumnNode*)pPhyNode->window.pTspk)->node.resType.precision};
H
Haojun Liao 已提交
1731 1732 1733 1734 1735 1736 1737 1738 1739

  STimeWindowAggSupp as = {
      .waterMark = pPhyNode->window.watermark,
      .calTrigger = pPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
  };

  ASSERT(as.calTrigger != STREAM_TRIGGER_MAX_DELAY);

L
Liu Jicong 已提交
1740
  pInfo->win = pTaskInfo->window;
1741 1742
  pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
  pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
H
Haojun Liao 已提交
1743
  pInfo->interval = interval;
L
Liu Jicong 已提交
1744
  pInfo->execModel = pTaskInfo->execModel;
H
Haojun Liao 已提交
1745
  pInfo->twAggSup = as;
1746
  pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock;
1747 1748 1749 1750

  if (pPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
H
Haojun Liao 已提交
1751
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
1752 1753 1754 1755
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
1756

H
Haojun Liao 已提交
1757 1758 1759 1760 1761
  code = filterInitFromNode((SNode*)pPhyNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

1762
  if (isStream) {
H
Haojun Liao 已提交
1763
    ASSERT(num > 0);
H
Haojun Liao 已提交
1764
    initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
1765
  }
1766

1767
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
H
Haojun Liao 已提交
1768
  pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, pInfo);
1769
  if (pInfo->timeWindowInterpo) {
1770
    pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
H
Haojun Liao 已提交
1771 1772 1773
    if (pInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
1774
  }
1775

1776
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
L
Liu Jicong 已提交
1777 1778
  setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED,
                  pInfo, pTaskInfo);
1779

1780
  pOperator->fpSet =
H
Haojun Liao 已提交
1781
      createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, NULL);
1782 1783 1784 1785 1786 1787 1788 1789

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1790
_error:
H
Haojun Liao 已提交
1791 1792 1793
  if (pInfo != NULL) {
    destroyIntervalOperatorInfo(pInfo);
  }
1794 1795 1796 1797 1798
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

1799
// todo handle multiple timeline cases. assume no timeline interweaving
1800 1801
static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1802
  SExprSupp*     pSup = &pOperator->exprSupp;
1803

1804
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1805 1806

  bool    masterScan = true;
1807
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822
  int64_t gid = pBlock->info.groupId;

  int64_t gap = pInfo->gap;

  if (!pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->winSup.prevTs = INT64_MIN;
  }

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

  // In case of ascending or descending order scan data, only one time window needs to be kepted for each table.
  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
1823 1824 1825
    if (gid != pRowSup->groupId || pInfo->winSup.prevTs == INT64_MIN) {
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
H
Haojun Liao 已提交
1826 1827
    } else if (((tsList[j] - pRowSup->prevTs >= 0) && (tsList[j] - pRowSup->prevTs <= gap)) ||
               ((pRowSup->prevTs - tsList[j] >= 0) && (pRowSup->prevTs - tsList[j] <= gap))) {
1828
      // The gap is less than the threshold, so it belongs to current session window that has been opened already.
1829
      doKeepTuple(pRowSup, tsList[j], gid);
1830 1831 1832 1833 1834 1835 1836 1837 1838 1839
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1840 1841
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1842
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1843
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1844 1845 1846 1847
      }

      // pInfo->numOfRows data belong to the current session window
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1848 1849
      doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                       pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1850 1851

      // here we start a new session window
1852 1853
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1854 1855 1856 1857 1858
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1859 1860
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1861
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1862
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1863 1864 1865
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
H
Haojun Liao 已提交
1866 1867
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
                   pBlock->info.rows, numOfOutput);
1868 1869
}

1870
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
1871 1872 1873 1874 1875 1876
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*          pBInfo = &pInfo->binfo;
1877
  SExprSupp*               pSup = &pOperator->exprSupp;
1878 1879

  if (pOperator->status == OP_RES_TO_RETURN) {
1880
    while (1) {
1881
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
H
Haojun Liao 已提交
1882
      doFilter(pBInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
1883

1884
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1885
      if (!hasRemain) {
H
Haojun Liao 已提交
1886
        setOperatorCompleted(pOperator);
1887 1888
        break;
      }
1889

1890 1891 1892 1893 1894
      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1895
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1896 1897
  }

1898 1899 1900
  int64_t st = taosGetTimestampUs();
  int32_t order = TSDB_ORDER_ASC;

1901 1902 1903
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1904
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1905 1906 1907 1908 1909
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1910
    setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
1911 1912
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1913 1914 1915
    doSessionWindowAggImpl(pOperator, pInfo, pBlock);
  }

1916 1917
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;

1918 1919 1920
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;

1921
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1922
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1923
  while (1) {
1924
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
H
Haojun Liao 已提交
1925
    doFilter(pBInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
1926

1927
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1928
    if (!hasRemain) {
H
Haojun Liao 已提交
1929
      setOperatorCompleted(pOperator);
1930 1931
      break;
    }
1932

1933 1934 1935 1936 1937
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1938
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1939 1940
}

1941
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
H
Haojun Liao 已提交
1942
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
1943 1944
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
H
Haojun Liao 已提交
1945

G
Ganlin Zhao 已提交
1946 1947
    SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);
    if (!colDataIsNull_s(pColInfoData, rowIndex)) {
H
Haojun Liao 已提交
1948
      pkey->isNull = false;
1949
      char* val = colDataGetData(pColInfoData, rowIndex);
1950 1951 1952 1953 1954
      if (!IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, pkey->bytes);
      } else {
        memcpy(pkey->pData, val, varDataLen(val));
      }
G
Ganlin Zhao 已提交
1955 1956
    } else {
      pkey->isNull = true;
H
Haojun Liao 已提交
1957 1958
    }
  }
1959 1960 1961 1962 1963 1964 1965 1966 1967

  pSliceInfo->isPrevRowSet = true;
}

static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

G
Ganlin Zhao 已提交
1968 1969
    SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i);
    if (!colDataIsNull_s(pColInfoData, rowIndex)) {
1970 1971
      pkey->isNull = false;
      char* val = colDataGetData(pColInfoData, rowIndex);
1972 1973 1974 1975 1976
      if (!IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, pkey->bytes);
      } else {
        memcpy(pkey->pData, val, varDataLen(val));
      }
G
Ganlin Zhao 已提交
1977 1978
    } else {
      pkey->isNull = true;
1979 1980 1981 1982
    }
  }

  pSliceInfo->isNextRowSet = true;
H
Haojun Liao 已提交
1983 1984
}

1985
static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
1986 1987 1988
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
1989 1990
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
    SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i);
1991

G
Ganlin Zhao 已提交
1992 1993 1994 1995
    // null value is represented by using key = INT64_MIN for now.
    // TODO: optimize to ignore null values for linear interpolation.
    if (!pLinearInfo->isStartSet) {
      if (!colDataIsNull_s(pColInfoData, rowIndex)) {
1996 1997
        pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
G
Ganlin Zhao 已提交
1998 1999 2000 2001
      }
      pLinearInfo->isStartSet = true;
    } else if (!pLinearInfo->isEndSet) {
      if (!colDataIsNull_s(pColInfoData, rowIndex)) {
2002 2003 2004
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
      }
G
Ganlin Zhao 已提交
2005
      pLinearInfo->isEndSet = true;
2006
    } else {
G
Ganlin Zhao 已提交
2007 2008 2009 2010
      pLinearInfo->start.key = pLinearInfo->end.key;
      memcpy(pLinearInfo->start.val, pLinearInfo->end.val, pLinearInfo->bytes);

      if (!colDataIsNull_s(pColInfoData, rowIndex)) {
2011 2012
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
G
Ganlin Zhao 已提交
2013 2014
      } else {
        pLinearInfo->end.key = INT64_MIN;
2015
      }
2016 2017 2018 2019 2020
    }
  }

}

G
Ganlin Zhao 已提交
2021
static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, bool beforeTs) {
2022
  int32_t rows = pResBlock->info.rows;
2023
  blockDataEnsureCapacity(pResBlock, rows + 1);
2024 2025 2026
  // todo set the correct primary timestamp column

  // output the result
2027
  bool hasInterp = true;
2028 2029 2030
  for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
    SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];

2031
    int32_t          dstSlot = pExprInfo->base.resSchema.slotId;
2032 2033
    SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

2034 2035 2036 2037 2038
    if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
      colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false);
      continue;
    }

G
Ganlin Zhao 已提交
2039
    int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
2040
    switch (pSliceInfo->fillType) {
2041
      case TSDB_FILL_NULL: {
2042 2043
        colDataAppendNULL(pDst, rows);
        break;
2044
      }
2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061

      case TSDB_FILL_SET_VALUE: {
        SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal;

        if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
          float v = 0;
          GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
          double v = 0;
          GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
          int64_t v = 0;
          GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        }
2062 2063
        break;
      }
2064

2065
      case TSDB_FILL_LINEAR: {
2066
        SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, srcSlot);
2067

dengyihao's avatar
dengyihao 已提交
2068 2069
        SPoint start = pLinearInfo->start;
        SPoint end = pLinearInfo->end;
2070 2071
        SPoint current = {.key = pSliceInfo->current};

G
Ganlin Zhao 已提交
2072 2073 2074 2075 2076
        // do not interpolate before ts range, only increate pSliceInfo->current
        if (beforeTs && !pLinearInfo->isEndSet) {
          return true;
        }

G
Ganlin Zhao 已提交
2077
        if (!pLinearInfo->isStartSet || !pLinearInfo->isEndSet) {
2078
          hasInterp = false;
2079 2080 2081
          break;
        }

G
Ganlin Zhao 已提交
2082
        if (start.key == INT64_MIN || end.key == INT64_MIN) {
2083
          colDataAppendNULL(pDst, rows);
G
Ganlin Zhao 已提交
2084
          break;
2085 2086
        }

G
Ganlin Zhao 已提交
2087
        current.val = taosMemoryCalloc(pLinearInfo->bytes, 1);
2088 2089
        taosGetLinearInterpolationVal(&current, pLinearInfo->type, &start, &end, pLinearInfo->type);
        colDataAppend(pDst, rows, (char*)current.val, false);
2090

2091
        taosMemoryFree(current.val);
2092
        break;
2093
      }
2094
      case TSDB_FILL_PREV: {
2095
        if (!pSliceInfo->isPrevRowSet) {
2096
          hasInterp = false;
2097 2098 2099
          break;
        }

2100
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
G
Ganlin Zhao 已提交
2101 2102 2103 2104 2105
        if (pkey->isNull == false) {
          colDataAppend(pDst, rows, pkey->pData, false);
        } else {
          colDataAppendNULL(pDst, rows);
        }
2106 2107
        break;
      }
2108 2109

      case TSDB_FILL_NEXT: {
2110
        if (!pSliceInfo->isNextRowSet) {
2111
          hasInterp = false;
2112 2113 2114
          break;
        }

2115
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot);
G
Ganlin Zhao 已提交
2116 2117 2118 2119 2120
        if (pkey->isNull == false) {
          colDataAppend(pDst, rows, pkey->pData, false);
        } else {
          colDataAppendNULL(pDst, rows);
        }
2121 2122
        break;
      }
2123 2124 2125 2126 2127 2128

      case TSDB_FILL_NONE:
      default:
        break;
    }
  }
2129 2130 2131 2132

  if (hasInterp) {
    pResBlock->info.rows += 1;
  }
2133 2134

  return hasInterp;
2135 2136
}

G
Ganlin Zhao 已提交
2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152
static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock,
                                  SSDataBlock* pSrcBlock, int32_t index) {
  blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1);
  for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
    SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];

    int32_t          dstSlot = pExprInfo->base.resSchema.slotId;
    SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

    if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
      colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false);
    } else {
      int32_t          srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
      SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot);

      if (colDataIsNull_s(pSrc, index)) {
G
Ganlin Zhao 已提交
2153 2154
        colDataAppendNULL(pDst, pResBlock->info.rows);
        continue;
G
Ganlin Zhao 已提交
2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166
      }

      char* v = colDataGetData(pSrc, index);
      colDataAppend(pDst, pResBlock->info.rows, v, false);
    }
  }

  pResBlock->info.rows += 1;
  return;
}


2167 2168 2169 2170 2171 2172 2173 2174 2175 2176
static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pPrevRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pPrevRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

2177
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pPrevRow, &key);
  }

2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217
  pInfo->isPrevRowSet = false;

  return TSDB_CODE_SUCCESS;
}

static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pNextRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pNextRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pNextRow, &key);
  }

  pInfo->isNextRowSet = false;

2218 2219 2220
  return TSDB_CODE_SUCCESS;
}

2221 2222 2223 2224 2225 2226
static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pLinearInfo != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo));
2227
  if (pInfo->pLinearInfo == NULL) {
2228 2229 2230 2231 2232 2233 2234
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

2235 2236
    SFillLinearInfo linearInfo = {0};
    linearInfo.start.key = INT64_MIN;
G
Ganlin Zhao 已提交
2237
    linearInfo.end.key = INT64_MIN;
2238
    linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes);
dengyihao's avatar
dengyihao 已提交
2239
    linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes);
G
Ganlin Zhao 已提交
2240 2241
    linearInfo.isStartSet = false;
    linearInfo.isEndSet = false;
dengyihao's avatar
dengyihao 已提交
2242
    linearInfo.type = pColInfo->info.type;
2243 2244
    linearInfo.bytes = pColInfo->info.bytes;
    taosArrayPush(pInfo->pLinearInfo, &linearInfo);
2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266
  }

  return TSDB_CODE_SUCCESS;
}

static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  int32_t code;
  code = initPrevRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initNextRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initFillLinearInfo(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

2267 2268 2269
  return TSDB_CODE_SUCCESS;
}

H
Haojun Liao 已提交
2270
static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
2271 2272 2273 2274
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

2275 2276
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

2277
  STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
2278 2279
  SSDataBlock*            pResBlock = pSliceInfo->pRes;
  SExprSupp*              pSup = &pOperator->exprSupp;
H
Haojun Liao 已提交
2280

2281 2282
  int32_t        order = TSDB_ORDER_ASC;
  SInterval*     pInterval = &pSliceInfo->interval;
2283 2284
  SOperatorInfo* downstream = pOperator->pDownstream[0];

2285 2286
  blockDataCleanup(pResBlock);

2287
  while (1) {
2288
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
2289 2290 2291 2292
    if (pBlock == NULL) {
      break;
    }

2293
    int32_t code = initKeeperInfo(pSliceInfo, pBlock);
2294
    if (code != TSDB_CODE_SUCCESS) {
2295
      T_LONG_JMP(pTaskInfo->env, code);
2296 2297
    }

2298
    // the pDataBlock are always the same one, no need to call this again
2299
    setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
H
Haojun Liao 已提交
2300

2301
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
2302 2303
    for (int32_t i = 0; i < pBlock->info.rows; ++i) {
      int64_t ts = *(int64_t*)colDataGetData(pTsCol, i);
H
Haojun Liao 已提交
2304

2305
      if (pSliceInfo->current > pSliceInfo->win.ekey) {
H
Haojun Liao 已提交
2306
        setOperatorCompleted(pOperator);
2307
        break;
2308 2309
      }

H
Haojun Liao 已提交
2310
      if (ts == pSliceInfo->current) {
G
Ganlin Zhao 已提交
2311
        addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i);
H
Haojun Liao 已提交
2312

2313
        doKeepPrevRows(pSliceInfo, pBlock, i);
2314
        doKeepLinearInfo(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2315

2316 2317 2318
        pSliceInfo->current =
            taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
        if (pSliceInfo->current > pSliceInfo->win.ekey) {
G
Ganlin Zhao 已提交
2319
          setOperatorCompleted(pOperator);
2320
          break;
2321
        }
H
Haojun Liao 已提交
2322
      } else if (ts < pSliceInfo->current) {
2323
        // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate
2324
        doKeepPrevRows(pSliceInfo, pBlock, i);
2325 2326 2327 2328 2329 2330 2331 2332
        doKeepLinearInfo(pSliceInfo, pBlock, i);

        if (i < pBlock->info.rows - 1) {
          // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
          doKeepNextRows(pSliceInfo, pBlock, i + 1);
          int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
          if (nextTs > pSliceInfo->current) {
            while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
G
Ganlin Zhao 已提交
2333
              if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) {
H
Haojun Liao 已提交
2334
                break;
2335 2336 2337
              } else {
                pSliceInfo->current =
                    taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
H
Haojun Liao 已提交
2338
              }
H
Haojun Liao 已提交
2339
            }
2340

2341
            if (pSliceInfo->current > pSliceInfo->win.ekey) {
G
Ganlin Zhao 已提交
2342
              setOperatorCompleted(pOperator);
2343
              break;
H
Haojun Liao 已提交
2344
            }
2345 2346
          } else {
            // ignore current row, and do nothing
H
Haojun Liao 已提交
2347
          }
2348 2349
        } else {  // it is the last row of current block
          doKeepPrevRows(pSliceInfo, pBlock, i);
2350 2351
        }
      } else {  // ts > pSliceInfo->current
2352
        // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
2353
        doKeepNextRows(pSliceInfo, pBlock, i);
2354
        doKeepLinearInfo(pSliceInfo, pBlock, i);
2355

2356
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
G
Ganlin Zhao 已提交
2357
          if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2358 2359 2360 2361 2362
            break;
          } else {
            pSliceInfo->current =
                taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          }
2363 2364
        }

2365 2366
        // add current row if timestamp match
        if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
G
Ganlin Zhao 已提交
2367
          addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i);
2368 2369
          doKeepPrevRows(pSliceInfo, pBlock, i);

2370 2371
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2372 2373
        }

2374
        if (pSliceInfo->current > pSliceInfo->win.ekey) {
H
Haojun Liao 已提交
2375
          setOperatorCompleted(pOperator);
2376
          break;
H
Haojun Liao 已提交
2377 2378 2379
        }
      }
    }
2380
  }
2381

2382 2383
  // check if need to interpolate after last datablock
  // except for fill(next), fill(linear)
dengyihao's avatar
dengyihao 已提交
2384 2385
  while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT &&
         pSliceInfo->fillType != TSDB_FILL_LINEAR) {
G
Ganlin Zhao 已提交
2386
    genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false);
2387 2388
    pSliceInfo->current =
        taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2389 2390 2391 2392
  }

  // restore the value
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
2393
  if (pResBlock->info.rows == 0) {
2394 2395 2396
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
2397 2398 2399
  return pResBlock->info.rows == 0 ? NULL : pResBlock;
}

2400
void destroyTimeSliceOperatorInfo(void* param) {
2401 2402 2403 2404 2405 2406 2407 2408
  STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param;

  pInfo->pRes = blockDataDestroy(pInfo->pRes);

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) {
    SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i);
    taosMemoryFree(pKey->pData);
  }
2409
  taosArrayDestroy(pInfo->pPrevRow);
2410 2411 2412 2413 2414 2415

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pNextRow); ++i) {
    SGroupKeys* pKey = taosArrayGet(pInfo->pNextRow, i);
    taosMemoryFree(pKey->pData);
  }
  taosArrayDestroy(pInfo->pNextRow);
2416 2417 2418 2419 2420 2421

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pLinearInfo); ++i) {
    SFillLinearInfo* pKey = taosArrayGet(pInfo->pLinearInfo, i);
    taosMemoryFree(pKey->start.val);
    taosMemoryFree(pKey->end.val);
  }
2422 2423
  taosArrayDestroy(pInfo->pLinearInfo);

2424
  taosMemoryFree(pInfo->pFillColInfo);
2425 2426 2427
  taosMemoryFreeClear(param);
}

2428
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
2429 2430 2431 2432 2433 2434
  STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pOperator == NULL || pInfo == NULL) {
    goto _error;
  }

2435
  SInterpFuncPhysiNode* pInterpPhyNode = (SInterpFuncPhysiNode*)pPhyNode;
2436
  SExprSupp*            pSup = &pOperator->exprSupp;
2437

2438
  int32_t    numOfExprs = 0;
2439
  SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
2440
  int32_t    code = initExprSupp(pSup, pExprInfo, numOfExprs);
H
Haojun Liao 已提交
2441 2442 2443 2444
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2445
  if (pInterpPhyNode->pExprs != NULL) {
2446
    int32_t    num = 0;
2447 2448 2449 2450 2451 2452 2453 2454 2455
    SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

  pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries);
  pInfo->fillType = convertFillType(pInterpPhyNode->fillMode);
2456
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2457

H
Haojun Liao 已提交
2458
  pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, NULL, 0, (SNodeListNode*)pInterpPhyNode->pFillValues);
2459
  pInfo->pLinearInfo = NULL;
L
Liu Jicong 已提交
2460 2461
  pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  pInfo->win = pInterpPhyNode->timeRange;
2462
  pInfo->interval.interval = pInterpPhyNode->interval;
L
Liu Jicong 已提交
2463
  pInfo->current = pInfo->win.skey;
H
Haojun Liao 已提交
2464

2465 2466
  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
    STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info;
H
Haojun Liao 已提交
2467 2468
    pScanInfo->base.cond.twindows = pInfo->win;
    pScanInfo->base.cond.type = TIMEWINDOW_RANGE_EXTERNAL;
2469
  }
2470

L
Liu Jicong 已提交
2471 2472 2473
  setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo,
                  pTaskInfo);
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL);
2474

2475 2476
  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);

H
Haojun Liao 已提交
2477
  code = appendDownstream(pOperator, &downstream, 1);
2478 2479
  return pOperator;

L
Liu Jicong 已提交
2480
_error:
2481 2482 2483 2484 2485 2486
  taosMemoryFree(pInfo);
  taosMemoryFree(pOperator);
  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
  return NULL;
}

2487 2488
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
                                             SExecTaskInfo* pTaskInfo) {
2489 2490 2491 2492 2493 2494
  SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

2495 2496 2497
  int32_t      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;

2498 2499 2500
  if (pStateNode->window.pExprs != NULL) {
    int32_t    numOfScalarExpr = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalarExpr);
H
Hongze Cheng 已提交
2501
    int32_t    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
2502 2503 2504 2505 2506
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

2507
  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
2508 2509 2510 2511 2512 2513 2514
  pInfo->stateKey.type = pInfo->stateCol.type;
  pInfo->stateKey.bytes = pInfo->stateCol.bytes;
  pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
  if (pInfo->stateKey.pData == NULL) {
    goto _error;
  }

H
Haojun Liao 已提交
2515 2516 2517 2518 2519
  int32_t code = filterInitFromNode((SNode*)pStateNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2520 2521
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

2522 2523
  int32_t    num = 0;
  SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num);
2524
  initResultSizeInfo(&pOperator->resultInfo, 4096);
H
Haojun Liao 已提交
2525 2526

  code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
2527 2528 2529 2530
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
2531
  SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc);
2532
  initBasicInfo(&pInfo->binfo, pResBlock);
2533
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2534

L
Liu Jicong 已提交
2535 2536
  pInfo->twAggSup =
      (STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType};
2537

2538 2539
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

X
Xiaoyu Wang 已提交
2540
  pInfo->tsSlotId = tsSlotId;
2541

L
Liu Jicong 已提交
2542 2543
  setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo,
                  pTaskInfo);
2544
  pOperator->fpSet =
H
Haojun Liao 已提交
2545
      createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, NULL);
2546

2547 2548 2549 2550 2551
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2552 2553
  return pOperator;

L
Liu Jicong 已提交
2554
_error:
H
Haojun Liao 已提交
2555 2556 2557 2558
  if (pInfo != NULL) {
    destroyStateWindowOperatorInfo(pInfo);
  }

2559 2560
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
2561 2562 2563
  return NULL;
}

2564
void destroySWindowOperatorInfo(void* param) {
2565
  SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
2566 2567 2568
  if (pInfo == NULL) {
    return;
  }
2569

2570
  cleanupBasicInfo(&pInfo->binfo);
H
Haojun Liao 已提交
2571 2572 2573 2574
  colDataDestroy(&pInfo->twAggSup.timeWindowData);

  cleanupAggSup(&pInfo->aggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
D
dapan1121 已提交
2575
  taosMemoryFreeClear(param);
2576 2577
}

H
Haojun Liao 已提交
2578
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode,
2579
                                            SExecTaskInfo* pTaskInfo) {
2580 2581 2582 2583 2584 2585
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

2586
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
2587
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2588

2589
  int32_t      numOfCols = 0;
H
Haojun Liao 已提交
2590 2591
  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc);
H
Haojun Liao 已提交
2592
  initBasicInfo(&pInfo->binfo, pResBlock);
H
Haojun Liao 已提交
2593

2594
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
2595 2596 2597 2598
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
2599 2600 2601 2602
  pInfo->twAggSup.waterMark = pSessionNode->window.watermark;
  pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType;
  pInfo->gap = pSessionNode->gap;

2603
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2604 2605
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

2606
  pInfo->tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
L
Liu Jicong 已提交
2607 2608 2609
  pInfo->binfo.pRes = pResBlock;
  pInfo->winSup.prevTs = INT64_MIN;
  pInfo->reptScan = false;
H
Haojun Liao 已提交
2610 2611 2612 2613
  code = filterInitFromNode((SNode*)pSessionNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
H
Haojun Liao 已提交
2614

L
Liu Jicong 已提交
2615 2616
  setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED,
                  pInfo, pTaskInfo);
2617
  pOperator->fpSet =
H
Haojun Liao 已提交
2618
      createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, NULL);
2619 2620
  pOperator->pTaskInfo = pTaskInfo;
  code = appendDownstream(pOperator, &downstream, 1);
2621 2622 2623 2624
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2625 2626
  return pOperator;

L
Liu Jicong 已提交
2627
_error:
2628
  destroySWindowOperatorInfo(pInfo);
2629 2630 2631
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
2632
}
5
54liuyao 已提交
2633

5
54liuyao 已提交
2634
void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
2635
                      SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) {
5
54liuyao 已提交
2636 2637
  for (int32_t k = 0; k < numOfOutput; ++k) {
    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
2638 2639 2640 2641 2642
      if (!pTimeWindowData) {
        continue;
      }

      SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);
L
Liu Jicong 已提交
2643 2644
      char*                p = GET_ROWCELL_INTERBUF(pEntryInfo);
      SColumnInfoData      idata = {0};
2645 2646 2647 2648 2649 2650 2651 2652
      idata.info.type = TSDB_DATA_TYPE_BIGINT;
      idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
      idata.pData = p;

      SScalarParam out = {.columnData = &idata};
      SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
      pDestCtx[k].sfp.process(&tw, 1, &out);
      pEntryInfo->numOfRes = 1;
L
Liu Jicong 已提交
2653
    } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
2654
      int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
5
54liuyao 已提交
2655 2656 2657
      if (code != TSDB_CODE_SUCCESS) {
        qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
        pTaskInfo->code = code;
2658
        T_LONG_JMP(pTaskInfo->env, code);
5
54liuyao 已提交
2659 2660 2661 2662 2663
      }
    }
  }
}

2664 2665
bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) {
  return TSDB_CODE_SUCCESS == streamStateGet(pState, pKey, NULL, 0);
2666 2667
}

5
54liuyao 已提交
2668
static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, SHashObj* pUpdatedMap) {
2669 2670 2671 2672
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
  int32_t                      size = taosArrayGetSize(pWinArray);
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
5
54liuyao 已提交
2673
  SExprSupp*                   pSup = &pOperator->exprSupp;
5
54liuyao 已提交
2674 2675 2676
  if (!pInfo->pChildren) {
    return;
  }
5
54liuyao 已提交
2677
  for (int32_t i = 0; i < size; i++) {
H
Haojun Liao 已提交
2678
    SWinKey*    pWinRes = taosArrayGet(pWinArray, i);
2679
    SResultRow* pCurResult = NULL;
2680
    STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval);
5
54liuyao 已提交
2681
    if (isDeletedStreamWindow(&parentWin, pWinRes->groupId, pInfo->pState, &pInfo->twAggSup)) {
2682 2683
      continue;
    }
2684

5
54liuyao 已提交
2685
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
2686
    int32_t num = 0;
5
54liuyao 已提交
2687
    for (int32_t j = 0; j < numOfChildren; j++) {
2688 2689 2690 2691
      SOperatorInfo*               pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SStreamIntervalOperatorInfo* pChInfo = pChildOp->info;
      SExprSupp*                   pChildSup = &pChildOp->exprSupp;
      if (!hasIntervalWindow(pChInfo->pState, pWinRes)) {
2692 2693
        continue;
      }
2694 2695 2696 2697 2698 2699 2700
      if (num == 0) {
        int32_t code = setOutputBuf(pInfo->pState, &parentWin, &pCurResult, pWinRes->groupId, pSup->pCtx, numOfOutput,
                                    pSup->rowEntryInfoOffset, &pInfo->aggSup);
        if (code != TSDB_CODE_SUCCESS || pCurResult == NULL) {
          T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
        }
      }
2701
      num++;
2702
      SResultRow* pChResult = NULL;
2703 2704
      setOutputBuf(pChInfo->pState, &parentWin, &pChResult, pWinRes->groupId, pChildSup->pCtx, pChildSup->numOfExprs,
                   pChildSup->rowEntryInfoOffset, &pChInfo->aggSup);
2705 2706
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true);
      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
2707
      releaseOutputBuf(pChInfo->pState, pWinRes, pChResult);
5
54liuyao 已提交
2708
    }
2709
    if (num > 0 && pUpdatedMap) {
2710 2711 2712
      saveWinResultInfo(pCurResult->win.skey, pWinRes->groupId, pUpdatedMap);
      saveOutputBuf(pInfo->pState, pWinRes, pCurResult, pInfo->aggSup.resultRowSize);
      releaseOutputBuf(pInfo->pState, pWinRes, pCurResult);
2713
    }
5
54liuyao 已提交
2714 2715 2716 2717 2718
  }
}

bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
  SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId);
2719
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
L
Liu Jicong 已提交
2720
                                                               GET_RES_WINDOW_KEY_LEN(sizeof(int64_t)));
5
54liuyao 已提交
2721 2722 2723
  return p1 == NULL;
}

2724
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) {
5
54liuyao 已提交
2725 2726
  if (pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {
    SWinKey key = {.ts = pWin->skey, .groupId = groupId};
5
54liuyao 已提交
2727
    if (streamStateGet(pState, &key, NULL, 0) == TSDB_CODE_SUCCESS) {
5
54liuyao 已提交
2728 2729
      return false;
    }
2730
    return true;
5
54liuyao 已提交
2731 2732 2733 2734
  }
  return false;
}

L
Liu Jicong 已提交
2735 2736 2737 2738
int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,
                        STimeWindow* pNextWin) {
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2739 2740 2741 2742
  int32_t prevEndPos = forwardRows - 1 + startPos;
  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
}

H
Haojun Liao 已提交
2743
void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {
5
54liuyao 已提交
2744 2745 2746 2747
  SArray* childIds = taosArrayInit(8, sizeof(int32_t));
  for (int32_t i = 0; i < size; i++) {
    taosArrayPush(childIds, &i);
  }
H
Haojun Liao 已提交
2748
  taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*));
5
54liuyao 已提交
2749 2750 2751 2752
}

static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

2753
static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) {
2754
  tSimpleHashClear(pInfo->aggSup.pResultRowHashTable);
5
54liuyao 已提交
2755
  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
2756
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
L
Liu Jicong 已提交
2757
  pInfo->aggSup.currentPageId = -1;
2758
  streamStateClear(pInfo->pState);
5
54liuyao 已提交
2759 2760
}

5
54liuyao 已提交
2761 2762 2763 2764
static void clearSpecialDataBlock(SSDataBlock* pBlock) {
  if (pBlock->info.rows <= 0) {
    return;
  }
5
54liuyao 已提交
2765 2766 2767
  blockDataCleanup(pBlock);
}

2768
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
5
54liuyao 已提交
2769 2770
  // ASSERT(pDest->info.capacity >= pSource->info.rows);
  blockDataEnsureCapacity(pDest, pSource->info.rows);
5
54liuyao 已提交
2771
  clearSpecialDataBlock(pDest);
5
54liuyao 已提交
2772 2773
  SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
  SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
2774

5
54liuyao 已提交
2775
  // copy timestamp column
2776 2777
  colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info);
  for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) {
5
54liuyao 已提交
2778 2779 2780
    SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
    colDataAppendNNULL(pCol, 0, pSource->info.rows);
  }
2781

5
54liuyao 已提交
2782
  pDest->info.rows = pSource->info.rows;
2783 2784
  pDest->info.groupId = pSource->info.groupId;
  pDest->info.type = pSource->info.type;
5
54liuyao 已提交
2785 2786 2787
  blockDataUpdateTsWindow(pDest, 0);
}

5
54liuyao 已提交
2788 2789 2790 2791 2792 2793
static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
  clearSpecialDataBlock(pBlock);
  int32_t size = taosArrayGetSize(array);
  if (size - (*pIndex) == 0) {
    return;
  }
L
Liu Jicong 已提交
2794
  blockDataEnsureCapacity(pBlock, size - (*pIndex));
5
54liuyao 已提交
2795
  ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
2796 2797 2798 2799 2800
  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
5
54liuyao 已提交
2801
  for (; (*pIndex) < size; (*pIndex)++) {
L
Liu Jicong 已提交
2802
    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));
5
54liuyao 已提交
2803 2804 2805
    colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
    colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
2806 2807
    colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
5
54liuyao 已提交
2808 2809 2810 2811 2812 2813 2814 2815 2816
    pBlock->info.rows++;
  }
  if ((*pIndex) == size) {
    *pIndex = 0;
    taosArrayClear(array);
  }
  blockDataUpdateTsWindow(pBlock, 0);
}

L
Liu Jicong 已提交
2817
void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
5
54liuyao 已提交
2818
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
L
Liu Jicong 已提交
2819
  TSKEY*           tsData = (TSKEY*)pStartCol->pData;
5
54liuyao 已提交
2820
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
L
Liu Jicong 已提交
2821 2822
  uint64_t*        groupIdData = (uint64_t*)pGroupCol->pData;
  int32_t          chId = getChildIndex(pBlock);
5
54liuyao 已提交
2823
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
H
Haojun Liao 已提交
2824 2825
    SWinKey winRes = {.ts = tsData[i], .groupId = groupIdData[i]};
    void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
2826
    if (chIds) {
L
Liu Jicong 已提交
2827
      SArray* chArray = *(SArray**)chIds;
5
54liuyao 已提交
2828 2829
      int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
      if (index != -1) {
5
54liuyao 已提交
2830
        qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId);
5
54liuyao 已提交
2831 2832 2833
        taosArrayRemove(chArray, index);
        if (taosArrayGetSize(chArray) == 0) {
          // pull data is over
5
54liuyao 已提交
2834
          taosArrayDestroy(chArray);
H
Haojun Liao 已提交
2835
          taosHashRemove(pMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
2836 2837 2838 2839 2840
        }
      }
    }
  }
}
5
54liuyao 已提交
2841

2842
static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) {
2843 2844
  int32_t size = taosArrayGetSize(wins);
  for (int32_t i = 0; i < size; i++) {
L
Liu Jicong 已提交
2845
    SWinKey*    winKey = taosArrayGet(wins, i);
2846
    STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
2847
    if (needDeleteWindowBuf(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) {
2848 2849 2850 2851 2852
      void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
      if (!chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId};
        // add pull data request
        savePullWindow(&pull, pInfo->pPullWins);
2853 2854 2855
        int32_t size1 = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, winKey, size1);
        qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, size1);
2856 2857 2858 2859 2860
      }
    }
  }
}

5
54liuyao 已提交
2861 2862 2863 2864 2865 2866
static void clearFunctionContext(SExprSupp* pSup) {
  for (int32_t i = 0; i < pSup->numOfExprs; i++) {
    pSup->pCtx[i].saveHandle.currentPage = -1;
  }
}

2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878
void doBuildResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  // set output datablock version
  pBlock->info.version = pTaskInfo->version;

  blockDataCleanup(pBlock);
  if (!hasRemainResults(pGroupResInfo)) {
    return;
  }

  // clear the existed group id
  pBlock->info.groupId = 0;
2879
  buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900
}

static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId,
                                    SHashObj* pUpdatedMap) {
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;

  SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*  pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*      pSup = &pOperatorInfo->exprSupp;
  int32_t         numOfOutput = pSup->numOfExprs;
  int32_t         step = 1;
  TSKEY*          tsCols = NULL;
  SResultRow*     pResult = NULL;
  int32_t         forwardRows = 0;

  ASSERT(pSDataBlock->pDataBlock != NULL);
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  tsCols = (int64_t*)pColDataInfo->pData;

  int32_t     startPos = 0;
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
5
54liuyao 已提交
2901 2902 2903 2904 2905 2906
  STimeWindow nextWin = {0};
  if (IS_FINAL_OP(pInfo)) {
    nextWin = getFinalTimeWindow(ts, &pInfo->interval);
  } else {
    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
  }
2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959
  while (1) {
    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
    if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
      if (startPos < 0) {
        break;
      }
      continue;
    }

    if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) {
      bool    ignore = true;
      SWinKey winRes = {
          .ts = nextWin.skey,
          .groupId = groupId,
      };
      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
      if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && !chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = groupId};
        // add pull data request
        savePullWindow(&pull, pInfo->pPullWins);
        int32_t size = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, &winRes, size);
        qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size);
      } else {
        int32_t index = -1;
        SArray* chArray = NULL;
        int32_t chId = 0;
        if (chIds) {
          chArray = *(void**)chIds;
          chId = getChildIndex(pSDataBlock);
          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
        }
        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {
          ignore = false;
        }
      }

      if (ignore) {
        startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
        if (startPos < 0) {
          break;
        }
        continue;
      }
    }

    int32_t code = setOutputBuf(pInfo->pState, &nextWin, &pResult, groupId, pSup->pCtx, numOfOutput,
                                pSup->rowEntryInfoOffset, &pInfo->aggSup);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

5
54liuyao 已提交
2960 2961 2962 2963 2964 2965
    if (IS_FINAL_OP(pInfo)) {
      forwardRows = 1;
    } else {
      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,
                                             NULL, TSDB_ORDER_ASC);
    }
2966 2967 2968
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
      saveWinResultInfo(pResult->win.skey, groupId, pUpdatedMap);
    }
5
54liuyao 已提交
2969 2970 2971 2972 2973 2974 2975 2976

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
      SWinKey key = {
          .ts = pResult->win.skey,
          .groupId = groupId,
      };
      tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0);
    }
2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pSDataBlock->info.rows, numOfOutput);
    SWinKey key = {
        .ts = nextWin.skey,
        .groupId = groupId,
    };
    saveOutputBuf(pInfo->pState, &key, pResult, pInfo->aggSup.resultRowSize);
    releaseOutputBuf(pInfo->pState, &key, pResult);
    if (pInfo->delKey.ts > key.ts) {
      pInfo->delKey = key;
    }
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
    startPos =
        getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);
    if (startPos < 0) {
      break;
    }
  }
}

5
54liuyao 已提交
2999
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
3000
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
3001
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
L
Liu Jicong 已提交
3002 3003 3004

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  TSKEY          maxTs = INT64_MIN;
5
54liuyao 已提交
3005
  TSKEY          minTs = INT64_MAX;
5
54liuyao 已提交
3006

3007 3008
  SExprSupp* pSup = &pOperator->exprSupp;

5
54liuyao 已提交
3009
  qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
L
Liu Jicong 已提交
3010

5
54liuyao 已提交
3011 3012 3013
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
3014 3015 3016 3017
    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
    if (pInfo->pPullDataRes->info.rows != 0) {
      // process the rest of the data
      ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3018
      printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3019 3020 3021
      return pInfo->pPullDataRes;
    }

3022
    doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
3023 3024 3025 3026 3027 3028
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
      return pInfo->pDelRes;
    }

3029
    doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
3030 3031 3032
    if (pInfo->binfo.pRes->info.rows != 0) {
      printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
      return pInfo->binfo.pRes;
5
54liuyao 已提交
3033
    }
5
54liuyao 已提交
3034

H
Haojun Liao 已提交
3035
    setOperatorCompleted(pOperator);
5
54liuyao 已提交
3036 3037 3038 3039 3040 3041
    if (!IS_FINAL_OP(pInfo)) {
      clearFunctionContext(&pOperator->exprSupp);
      // semi interval operator clear disk buffer
      clearStreamIntervalOperator(pInfo);
      qDebug("===stream===clear semi operator");
    } else {
3042 3043
      deleteIntervalDiscBuf(pInfo->pState, pInfo->pPullDataMap, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark,
                            &pInfo->interval, &pInfo->delKey);
L
Liu Jicong 已提交
3044
      streamStateCommit(pTaskInfo->streamInfo.pState);
5
54liuyao 已提交
3045 3046
    }
    return NULL;
5
54liuyao 已提交
3047
  } else {
5
54liuyao 已提交
3048
    if (!IS_FINAL_OP(pInfo)) {
3049
      doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
3050 3051 3052 3053 3054 3055
      if (pInfo->pDelRes->info.rows != 0) {
        // process the rest of the data
        printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
        return pInfo->pDelRes;
      }

3056
      doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
3057
      if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3058
        printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3059 3060
        return pInfo->binfo.pRes;
      }
5
54liuyao 已提交
3061
    }
5
54liuyao 已提交
3062 3063
  }

5
54liuyao 已提交
3064 3065 3066
  SArray*    pUpdated = taosArrayInit(4, POINTER_BYTES);
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*  pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
5
54liuyao 已提交
3067 3068 3069
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
3070
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
3071
      qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3072 3073
      break;
    }
5
54liuyao 已提交
3074
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv");
3075

H
Haojun Liao 已提交
3076 3077
    ASSERT(pBlock->info.type != STREAM_INVERT);
    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
5
54liuyao 已提交
3078
      pInfo->binfo.pRes->info.type = pBlock->info.type;
3079 3080
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
               pBlock->info.type == STREAM_CLEAR) {
3081
      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
3082
      doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pUpdatedMap);
3083
      if (IS_FINAL_OP(pInfo)) {
3084 3085 3086 3087
        int32_t                      childIndex = getChildIndex(pBlock);
        SOperatorInfo*               pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info;
        SExprSupp*                   pChildSup = &pChildOp->exprSupp;
3088
        doDeleteWindows(pChildOp, &pChildInfo->interval, pBlock, NULL, NULL);
5
54liuyao 已提交
3089
        rebuildIntervalWindow(pOperator, delWins, pUpdatedMap);
3090 3091 3092
        addRetriveWindow(delWins, pInfo);
        taosArrayAddAll(pInfo->pDelWins, delWins);
        taosArrayDestroy(delWins);
3093 3094
        continue;
      }
3095 3096 3097
      removeResults(delWins, pUpdatedMap);
      taosArrayAddAll(pInfo->pDelWins, delWins);
      taosArrayDestroy(delWins);
3098
      break;
5
54liuyao 已提交
3099
    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
3100
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
5
54liuyao 已提交
3101
      continue;
5
54liuyao 已提交
3102
    } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
3103
      doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pUpdatedMap);
5
54liuyao 已提交
3104 3105 3106 3107
      if (taosArrayGetSize(pUpdated) > 0) {
        break;
      }
      continue;
L
Liu Jicong 已提交
3108 3109
    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) {
      processPullOver(pBlock, pInfo->pPullDataMap);
5
54liuyao 已提交
3110
      continue;
5
54liuyao 已提交
3111
    }
5
54liuyao 已提交
3112

5
54liuyao 已提交
3113 3114 3115 3116
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
3117
    setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
3118
    doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
5
54liuyao 已提交
3119
    if (IS_FINAL_OP(pInfo)) {
S
shenglian zhou 已提交
3120
      int32_t chIndex = getChildIndex(pBlock);
5
54liuyao 已提交
3121 3122 3123 3124 3125
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
        SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
        if (!pChildOp) {
3126
          T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3127
        }
3128
        SStreamIntervalOperatorInfo* pTmpInfo = pChildOp->info;
3129
        pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3130
        taosArrayPush(pInfo->pChildren, &pChildOp);
5
54liuyao 已提交
3131
        qDebug("===stream===add child, id:%d", chIndex);
5
54liuyao 已提交
3132
      }
3133 3134
      SOperatorInfo*               pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
      SStreamIntervalOperatorInfo* pChInfo = pChildOp->info;
3135
      setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
3136
      doStreamIntervalAggImpl(pChildOp, pBlock, pBlock->info.groupId, NULL);
5
54liuyao 已提交
3137
    }
5
54liuyao 已提交
3138 3139 3140
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
    maxTs = TMAX(maxTs, pBlock->info.watermark);
    minTs = TMIN(minTs, pBlock->info.window.skey);
5
54liuyao 已提交
3141
  }
S
shenglian zhou 已提交
3142

3143
  removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
5
54liuyao 已提交
3144
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
3145
  pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs);
5
54liuyao 已提交
3146
  if (IS_FINAL_OP(pInfo)) {
3147
    closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval,
3148
                              pInfo->pPullDataMap, pUpdatedMap, pInfo->pDelWins, pOperator);
3149
    closeChildIntervalWindow(pOperator, pInfo->pChildren, pInfo->twAggSup.maxTs);
5
54liuyao 已提交
3150
  }
3151
  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;
5
54liuyao 已提交
3152

5
54liuyao 已提交
3153 3154 3155 3156 3157 3158 3159
  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosHashCleanup(pUpdatedMap);
  taosArraySort(pUpdated, resultrowComparAsc);

5
54liuyao 已提交
3160 3161
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
3162 3163 3164 3165 3166

  doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
  if (pInfo->pPullDataRes->info.rows != 0) {
    // process the rest of the data
    ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3167
    printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3168 3169 3170
    return pInfo->pPullDataRes;
  }

3171
  doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
5
54liuyao 已提交
3172 3173 3174 3175 3176 3177
  if (pInfo->pDelRes->info.rows != 0) {
    // process the rest of the data
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
    return pInfo->pDelRes;
  }

3178
  doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
3179
  if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3180
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3181 3182 3183 3184 3185 3186
    return pInfo->binfo.pRes;
  }

  return NULL;
}

S
shenglian zhou 已提交
3187 3188
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                     SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
3189 3190 3191
  SIntervalPhysiNode*          pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3192 3193 3194
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
3195

3196
  pOperator->pTaskInfo = pTaskInfo;
S
shenglian zhou 已提交
3197 3198 3199 3200 3201 3202 3203 3204
  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                .sliding = pIntervalPhyNode->sliding,
                                .intervalUnit = pIntervalPhyNode->intervalUnit,
                                .slidingUnit = pIntervalPhyNode->slidingUnit,
                                .offset = pIntervalPhyNode->offset,
                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pIntervalPhyNode->window.watermark,
5
54liuyao 已提交
3205 3206
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
3207
      .minTs = INT64_MAX,
5
54liuyao 已提交
3208 3209 3210
      // for test 315360000000
      .deleteMark = 1000LL * 60LL * 60LL * 24LL * 365LL * 10LL,
      // .deleteMark = INT64_MAX,
L
Liu Jicong 已提交
3211 3212
      .deleteMarkSaved = 0,
      .calTriggerSaved = 0,
S
shenglian zhou 已提交
3213
  };
3214
  ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY);
5
54liuyao 已提交
3215 3216
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
3217
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3218 3219 3220 3221 3222 3223 3224 3225 3226
  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

S
shenglian zhou 已提交
3227 3228
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5
54liuyao 已提交
3229
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
5
54liuyao 已提交
3230
  initBasicInfo(&pInfo->binfo, pResBlock);
3231 3232

  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
3233 3234 3235 3236
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
3237
  initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs);
3238

3239
  ASSERT(numOfCols > 0);
5
54liuyao 已提交
3240
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
3241

3242 3243 3244 3245
  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
  *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
  streamStateSetNumber(pInfo->pState, -1);

3246
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3247 3248
  pInfo->pChildren = NULL;
  if (numOfChild > 0) {
3249
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
5
54liuyao 已提交
3250 3251 3252
    for (int32_t i = 0; i < numOfChild; i++) {
      SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp) {
3253
        SStreamIntervalOperatorInfo* pChInfo = pChildOp->info;
3254
        pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3255
        taosArrayPush(pInfo->pChildren, &pChildOp);
3256
        streamStateSetNumber(pChInfo->pState, i);
5
54liuyao 已提交
3257 3258 3259 3260 3261
        continue;
      }
      goto _error;
    }
  }
5
54liuyao 已提交
3262

3263
  pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
5
54liuyao 已提交
3264

5
54liuyao 已提交
3265 3266 3267 3268
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
    pInfo->isFinal = true;
    pOperator->name = "StreamFinalIntervalOperator";
  } else {
5
54liuyao 已提交
3269
    // semi interval operator does not catch result
5
54liuyao 已提交
3270 3271
    pInfo->isFinal = false;
    pOperator->name = "StreamSemiIntervalOperator";
H
Haojun Liao 已提交
3272
    ASSERT(pInfo->aggSup.currentPageId == -1);
5
54liuyao 已提交
3273 3274
  }

5
54liuyao 已提交
3275
  if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
5
54liuyao 已提交
3276 3277
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }
5
54liuyao 已提交
3278 3279 3280 3281
  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
  pInfo->pullIndex = 0;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
3282
  pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
5
54liuyao 已提交
3283
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
3284
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3285
  pInfo->delIndex = 0;
H
Haojun Liao 已提交
3286
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
3287 3288
  pInfo->delKey.ts = INT64_MAX;
  pInfo->delKey.groupId = 0;
5
54liuyao 已提交
3289

5
54liuyao 已提交
3290
  pOperator->operatorType = pPhyNode->type;
5
54liuyao 已提交
3291 3292 3293 3294
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;

S
shenglian zhou 已提交
3295
  pOperator->fpSet =
H
Haojun Liao 已提交
3296
      createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL);
3297
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
5
54liuyao 已提交
3298
    initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
3299
  }
5
54liuyao 已提交
3300 3301 3302 3303 3304 3305 3306 3307
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
3308
  destroyStreamFinalIntervalOperatorInfo(pInfo);
5
54liuyao 已提交
3309 3310 3311
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
5
54liuyao 已提交
3312
}
5
54liuyao 已提交
3313 3314

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
5
54liuyao 已提交
3315
  tSimpleHashCleanup(pSup->pResultRows);
5
54liuyao 已提交
3316 3317
  destroyDiskbasedBuf(pSup->pResultBuf);
  blockDataDestroy(pSup->pScanBlock);
5
54liuyao 已提交
3318 3319
  taosMemoryFreeClear(pSup->pState);
  taosMemoryFreeClear(pSup->pDummyCtx);
5
54liuyao 已提交
3320 3321
}

3322
void destroyStreamSessionAggOperatorInfo(void* param) {
5
54liuyao 已提交
3323
  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;
3324
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
3325
  destroyStreamAggSupporter(&pInfo->streamAggSup);
5
54liuyao 已提交
3326

3327 3328 3329
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
3330 3331
      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);
      destroyOperatorInfo(pChild);
3332
    }
5
54liuyao 已提交
3333
    taosArrayDestroy(pInfo->pChildren);
3334
  }
5
54liuyao 已提交
3335 3336 3337 3338
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  blockDataDestroy(pInfo->pWinBlock);
  blockDataDestroy(pInfo->pUpdateRes);
5
54liuyao 已提交
3339
  tSimpleHashCleanup(pInfo->pStDeleted);
3340

D
dapan1121 已提交
3341
  taosMemoryFreeClear(param);
5
54liuyao 已提交
3342 3343
}

3344 3345
int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,
                        SSDataBlock* pResultBlock) {
H
Haojun Liao 已提交
3346
  initBasicInfo(pBasicInfo, pResultBlock);
3347 3348 3349 3350
  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
  if (code != TSDB_CODE_SUCCESS) {
    return code;
  }
3351

H
Haojun Liao 已提交
3352
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
5
54liuyao 已提交
3353
  for (int32_t i = 0; i < numOfCols; ++i) {
3354
    pSup->pCtx[i].saveHandle.pBuf = NULL;
5
54liuyao 已提交
3355
  }
3356

3357
  ASSERT(numOfCols > 0);
5
54liuyao 已提交
3358 3359 3360 3361 3362 3363 3364 3365
  return TSDB_CODE_SUCCESS;
}

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {
  for (int i = 0; i < nums; i++) {
    pDummy[i].functionId = pCtx[i].functionId;
  }
}
5
54liuyao 已提交
3366

5
54liuyao 已提交
3367 3368
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
                    STimeWindowAggSupp* pTwSup) {
3369 3370 3371 3372 3373 3374
  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
    pScanInfo->tsColIndex = tsColIndex;
  }

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
5
54liuyao 已提交
3375
    initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup);
3376 3377
    return;
  }
3378
  SStreamScanInfo* pScanInfo = downstream->info;
5
54liuyao 已提交
3379
  pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
5
54liuyao 已提交
3380
  if (!pScanInfo->pUpdateInfo) {
5
54liuyao 已提交
3381
    pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark);
5
54liuyao 已提交
3382
  }
5
54liuyao 已提交
3383
  pScanInfo->twAggSup = *pTwSup;
5
54liuyao 已提交
3384 3385
}

5
54liuyao 已提交
3386 3387 3388 3389 3390 3391 3392 3393 3394 3395
int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap,
                               SStreamState* pState, int32_t keySize, int16_t keyType) {
  pSup->resultRowSize = keySize + getResultRowSize(pCtx, numOfOutput);
  pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR);
  pSup->gap = gap;
  pSup->stateKeySize = keySize;
  pSup->stateKeyType = keyType;
  pSup->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx));
  if (pSup->pDummyCtx == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
3396
  }
H
Haojun Liao 已提交
3397

5
54liuyao 已提交
3398 3399 3400 3401
  initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput);
  pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState));
  *(pSup->pState) = *pState;
  streamStateSetNumber(pSup->pState, -1);
3402

5
54liuyao 已提交
3403 3404
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pSup->pResultRows = tSimpleHashInit(32, hashFn);
X
Xiaoyu Wang 已提交
3405

5
54liuyao 已提交
3406 3407 3408
  int32_t pageSize = 4096;
  while (pageSize < pSup->resultRowSize * 4) {
    pageSize <<= 1u;
5
54liuyao 已提交
3409
  }
5
54liuyao 已提交
3410 3411 3412 3413
  // at least four pages need to be in buffer
  int32_t bufSize = 4096 * 256;
  if (bufSize <= pageSize) {
    bufSize = pageSize * 4;
5
54liuyao 已提交
3414
  }
5
54liuyao 已提交
3415 3416 3417 3418
  if (!osTempSpaceAvailable()) {
    terrno = TSDB_CODE_NO_AVAIL_DISK;
    qError("Init stream agg supporter failed since %s", terrstr(terrno));
    return terrno;
5
54liuyao 已提交
3419
  }
5
54liuyao 已提交
3420 3421 3422
  int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir);
  for (int32_t i = 0; i < numOfOutput; ++i) {
    pCtx[i].saveHandle.pBuf = pSup->pResultBuf;
5
54liuyao 已提交
3423 3424
  }

5
54liuyao 已提交
3425
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
3426
}
5
54liuyao 已提交
3427 3428

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {
5
54liuyao 已提交
3429
  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {
5
54liuyao 已提交
3430 3431 3432 3433 3434
    return true;
  }
  return false;
}

5
54liuyao 已提交
3435 3436
bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) {
  return isInTimeWindow(&pWinInfo->sessionWin.win, ts, gap);
5
54liuyao 已提交
3437 3438
}

5
54liuyao 已提交
3439 3440 3441 3442 3443
void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
                         SSessionKey* pKey) {
  pKey->win.skey = startTs;
  pKey->win.ekey = endTs;
  pKey->groupId = groupId;
3444
  int32_t code = streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey);
5
54liuyao 已提交
3445 3446
  if (code != TSDB_CODE_SUCCESS) {
    SET_SESSION_WIN_KEY_INVALID(pKey);
3447 3448 3449
  }
}

5
54liuyao 已提交
3450
bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->sessionWin.win.skey == 0; }
5
54liuyao 已提交
3451

5
54liuyao 已提交
3452 3453 3454
void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
                         SResultWindowInfo* pCurWin) {
  pCurWin->sessionWin.groupId = groupId;
3455 3456
  pCurWin->sessionWin.win.skey = startTs;
  pCurWin->sessionWin.win.ekey = endTs;
5
54liuyao 已提交
3457
  int32_t size = pAggSup->resultRowSize;
3458 3459
  int32_t code =
      streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, &pCurWin->pOutputBuf, &size);
5
54liuyao 已提交
3460 3461 3462 3463 3464
  if (code == TSDB_CODE_SUCCESS) {
    pCurWin->isOutput = true;
  } else {
    pCurWin->sessionWin.win.skey = startTs;
    pCurWin->sessionWin.win.ekey = endTs;
5
54liuyao 已提交
3465
  }
5
54liuyao 已提交
3466
}
5
54liuyao 已提交
3467

5
54liuyao 已提交
3468 3469
int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) {
  int32_t size = 0;
3470
  int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size);
5
54liuyao 已提交
3471 3472
  if (code != TSDB_CODE_SUCCESS) {
    return code;
5
54liuyao 已提交
3473
  }
5
54liuyao 已提交
3474 3475 3476 3477 3478 3479
  streamStateCurNext(pAggSup->pState, pCur);
  return TSDB_CODE_SUCCESS;
}
void saveDeleteInfo(SArray* pWins, SSessionKey key) {
  // key.win.ekey = key.win.skey;
  taosArrayPush(pWins, &key);
5
54liuyao 已提交
3480 3481
}

5
54liuyao 已提交
3482 3483 3484 3485
void saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) {
  key.win.ekey = key.win.skey;
  tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0);
}
3486

5
54liuyao 已提交
3487 3488 3489 3490 3491
static void removeSessionResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey key) {
  key.win.ekey = key.win.skey;
  tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));
  tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey));
}
5
54liuyao 已提交
3492

5
54liuyao 已提交
3493 3494 3495 3496 3497
static void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey) {
  *pHashKey = *pKey;
  pHashKey->win.ekey = pKey->win.skey;
}

5
54liuyao 已提交
3498 3499 3500
static void removeSessionResults(SSHashObj* pHashMap, SArray* pWins) {
  if (tSimpleHashGetSize(pHashMap) == 0) {
    return;
5
54liuyao 已提交
3501
  }
5
54liuyao 已提交
3502 3503 3504 3505
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
    SSessionKey* pWin = taosArrayGet(pWins, i);
    if (!pWin) continue;
5
54liuyao 已提交
3506 3507
    SSessionKey key = {0};
    getSessionHashKey(pWin, &key);
5
54liuyao 已提交
3508
    tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));
5
54liuyao 已提交
3509 3510 3511
  }
}

dengyihao's avatar
dengyihao 已提交
3512
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId,
5
54liuyao 已提交
3513 3514
                                int32_t rows, int32_t start, int64_t gap, SSHashObj* pResultRows, SSHashObj* pStUpdated,
                                SSHashObj* pStDeleted) {
5
54liuyao 已提交
3515
  for (int32_t i = start; i < rows; ++i) {
3516
    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {
5
54liuyao 已提交
3517 3518
      return i - start;
    }
5
54liuyao 已提交
3519
    if (pWinInfo->sessionWin.win.skey > pStartTs[i]) {
5
54liuyao 已提交
3520
      if (pStDeleted && pWinInfo->isOutput) {
5
54liuyao 已提交
3521
        saveDeleteRes(pStDeleted, pWinInfo->sessionWin);
5
54liuyao 已提交
3522
      }
5
54liuyao 已提交
3523 3524
      removeSessionResult(pStUpdated, pResultRows, pWinInfo->sessionWin);
      pWinInfo->sessionWin.win.skey = pStartTs[i];
5
54liuyao 已提交
3525
    }
5
54liuyao 已提交
3526
    pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pStartTs[i]);
5
54liuyao 已提交
3527
    if (pEndTs) {
5
54liuyao 已提交
3528
      pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pEndTs[i]);
5
54liuyao 已提交
3529 3530 3531 3532 3533
    }
  }
  return rows - start;
}

5
54liuyao 已提交
3534 3535 3536 3537
static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,
                                    int32_t numOfOutput, int32_t* rowEntryInfoOffset) {
  ASSERT(pWinInfo->sessionWin.win.skey <= pWinInfo->sessionWin.win.ekey);
  *pResult = (SResultRow*)pWinInfo->pOutputBuf;
5
54liuyao 已提交
3538
  // set time window for current result
5
54liuyao 已提交
3539
  (*pResult)->win = pWinInfo->sessionWin.win;
3540
  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
5
54liuyao 已提交
3541 3542 3543
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
3544 3545 3546
static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult,
                                  int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput,
                                  SOperatorInfo* pOperator) {
3547
  SExprSupp*     pSup = &pOperator->exprSupp;
3548
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3549
  int32_t        code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
3550 3551 3552
  if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }
5
54liuyao 已提交
3553 3554
  updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, false);
  doApplyFunctions(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput);
5
54liuyao 已提交
3555 3556 3557
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
3558 3559
static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) {
  streamStateSessionDel(pAggSup->pState, pKey);
5
54liuyao 已提交
3560 3561 3562
  SSessionKey hashKey = {0};
  getSessionHashKey(pKey, &hashKey);
  tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey));
5
54liuyao 已提交
3563 3564 3565 3566 3567 3568 3569 3570 3571 3572
  return true;
}

static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) {
  void* pVal = tSimpleHashGet(pStUpdated, &pWinInfo->sessionWin, sizeof(SSessionKey));
  if (pVal) {
    SResultWindowInfo* pWin = pVal;
    pWinInfo->isOutput = pWin->isOutput;
  }
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
3573 3574
}

5
54liuyao 已提交
3575 3576 3577 3578 3579 3580 3581
SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,
                                       SResultWindowInfo* pNextWin) {
  SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin);
  pNextWin->isOutput = true;
  setSessionWinOutputInfo(pStUpdated, pNextWin);
  int32_t size = 0;
  pNextWin->sessionWin = pCurWin->sessionWin;
3582
  int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size);
5
54liuyao 已提交
3583 3584 3585 3586
  if (code != TSDB_CODE_SUCCESS) {
    SET_SESSION_WIN_INVALID(*pNextWin);
  }
  return pCur;
5
54liuyao 已提交
3587 3588
}

5
54liuyao 已提交
3589 3590 3591 3592 3593 3594 3595 3596 3597
static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated,
                                 SSHashObj* pStDeleted) {
  SExprSupp*                     pSup = &pOperator->exprSupp;
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SResultRow*                    pCurResult = NULL;
  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;
  initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
3598
  // Just look for the window behind StartIndex
5
54liuyao 已提交
3599 3600 3601 3602 3603 3604
  while (1) {
    SResultWindowInfo winInfo = {0};
    SStreamStateCur*  pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo);
    if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) {
      streamStateFreeCur(pCur);
      break;
5
54liuyao 已提交
3605
    }
5
54liuyao 已提交
3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617
    SResultRow* pWinResult = NULL;
    initSessionOutputBuf(&winInfo, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset);
    pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, winInfo.sessionWin.win.ekey);
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->sessionWin.win, true);
    compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
    tSimpleHashRemove(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey));
    if (winInfo.isOutput && pStDeleted) {
      saveDeleteRes(pStDeleted, winInfo.sessionWin);
    }
    removeSessionResult(pStUpdated, pAggSup->pResultRows, winInfo.sessionWin);
    doDeleteSessionWindow(pAggSup, &winInfo.sessionWin);
    streamStateFreeCur(pCur);
5
54liuyao 已提交
3618 3619 3620
  }
}

5
54liuyao 已提交
3621 3622 3623
int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) {
  saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize);
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
3624 3625
}

5
54liuyao 已提交
3626 3627
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated,
                                   SSHashObj* pStDeleted, bool hasEndTs) {
X
Xiaoyu Wang 已提交
3628
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3629
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
3630
  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
5
54liuyao 已提交
3631
  uint64_t                       groupId = pSDataBlock->info.groupId;
X
Xiaoyu Wang 已提交
3632
  int64_t                        code = TSDB_CODE_SUCCESS;
5
54liuyao 已提交
3633 3634 3635
  SResultRow*                    pResult = NULL;
  int32_t                        rows = pSDataBlock->info.rows;
  int32_t                        winRows = 0;
X
Xiaoyu Wang 已提交
3636

5
54liuyao 已提交
3637
  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
3638
  TSKEY*           startTsCols = (int64_t*)pStartTsCol->pData;
5
54liuyao 已提交
3639 3640 3641
  SColumnInfoData* pEndTsCol = NULL;
  if (hasEndTs) {
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);
5
54liuyao 已提交
3642
  } else {
5
54liuyao 已提交
3643
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
3644
  }
X
Xiaoyu Wang 已提交
3645

5
54liuyao 已提交
3646
  TSKEY*               endTsCols = (int64_t*)pEndTsCol->pData;
5
54liuyao 已提交
3647
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
5
54liuyao 已提交
3648
  for (int32_t i = 0; i < rows;) {
5
54liuyao 已提交
3649
    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
3650 3651 3652
      i++;
      continue;
    }
5
54liuyao 已提交
3653 3654 3655 3656 3657
    SResultWindowInfo winInfo = {0};
    setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo);
    setSessionWinOutputInfo(pStUpdated, &winInfo);
    winRows = updateSessionWindowInfo(&winInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap,
                                      pAggSup->pResultRows, pStUpdated, pStDeleted);
5
54liuyao 已提交
3658 3659 3660 3661 3662
    // coverity scan error
    if (!winInfo.pOutputBuf) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
  
5
54liuyao 已提交
3663 3664
    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput,
                              pOperator);
5
54liuyao 已提交
3665
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
3666
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3667
    }
5
54liuyao 已提交
3668 3669
    compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted);
    saveSessionOutputBuf(pAggSup, &winInfo);
5
54liuyao 已提交
3670 3671

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
5
54liuyao 已提交
3672
      code = saveResult(winInfo, pStUpdated);
5
54liuyao 已提交
3673
      if (code != TSDB_CODE_SUCCESS) {
3674
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3675
      }
5
54liuyao 已提交
3676
    }
5
54liuyao 已提交
3677
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
3678 3679
      SSessionKey key = {0};
      getSessionHashKey(&winInfo.sessionWin, &key);
5
54liuyao 已提交
3680 3681 3682
      tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));
    }

5
54liuyao 已提交
3683 3684 3685 3686
    i += winRows;
  }
}

5
54liuyao 已提交
3687
void deleteWindow(SArray* pWinInfos, int32_t index, FDelete fp) {
5
54liuyao 已提交
3688
  ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos));
5
54liuyao 已提交
3689 3690 3691 3692
  if (fp) {
    void* ptr = taosArrayGet(pWinInfos, index);
    fp(ptr);
  }
5
54liuyao 已提交
3693 3694 3695
  taosArrayRemove(pWinInfos, index);
}

5
54liuyao 已提交
3696
static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) {
5
54liuyao 已提交
3697
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
3698
  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;
5
54liuyao 已提交
3699
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
3700
  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;
3701
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
3702
  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;
5
54liuyao 已提交
3703
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
5
54liuyao 已提交
3704 3705 3706 3707
    while (1) {
      SSessionKey curWin = {0};
      getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], &curWin);
      if (IS_INVALID_SESSION_WIN_KEY(curWin)) {
3708 3709
        break;
      }
5
54liuyao 已提交
3710 3711 3712 3713
      doDeleteSessionWindow(pAggSup, &curWin);
      if (result) {
        saveDeleteInfo(result, curWin);
      }
3714
    }
5
54liuyao 已提交
3715 3716 3717
  }
}

5
54liuyao 已提交
3718 3719 3720 3721 3722 3723 3724 3725
static inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) {
  SSessionKey* pWin1 = (SSessionKey*)pKey1;
  SSessionKey* pWin2 = (SSessionKey*)pKey2;

  if (pWin1->groupId > pWin2->groupId) {
    return 1;
  } else if (pWin1->groupId < pWin2->groupId) {
    return -1;
5
54liuyao 已提交
3726 3727
  }

5
54liuyao 已提交
3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749
  if (pWin1->win.skey > pWin2->win.skey) {
    return 1;
  } else if (pWin1->win.skey < pWin2->win.skey) {
    return -1;
  }

  return 0;
}

static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) {
  void*   pIte = NULL;
  size_t  keyLen = 0;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pStUpdated, pIte, &iter)) != NULL) {
    void* key = tSimpleHashGetKey(pIte, &keyLen);
    ASSERT(keyLen == sizeof(SSessionKey));
    taosArrayPush(pUpdated, key);
  }
  taosArraySort(pUpdated, sessionKeyCompareAsc);
  return TSDB_CODE_SUCCESS;
}

3750
void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
5
54liuyao 已提交
3751 3752 3753 3754
  blockDataCleanup(pBlock);
  int32_t size = tSimpleHashGetSize(pStDeleted);
  if (size == 0) {
    return;
3755 3756
  }
  blockDataEnsureCapacity(pBlock, size);
5
54liuyao 已提交
3757 3758 3759 3760 3761 3762 3763
  size_t  keyLen = 0;
  int32_t iter = 0;
  while (((*Ite) = tSimpleHashIterate(pStDeleted, *Ite, &iter)) != NULL) {
    if (pBlock->info.rows + 1 > pBlock->info.capacity) {
      break;
    }
    SSessionKey*     res = tSimpleHashGetKey(*Ite, &keyLen);
3764
    SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
5
54liuyao 已提交
3765
    colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->win.skey, false);
3766
    SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
5
54liuyao 已提交
3767
    colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->win.skey, false);
3768 3769
    SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
    colDataAppendNULL(pUidCol, pBlock->info.rows);
5
54liuyao 已提交
3770 3771
    SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
    colDataAppend(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false);
3772 3773 3774 3775
    SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
    colDataAppendNULL(pCalStCol, pBlock->info.rows);
    SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
    colDataAppendNULL(pCalEdCol, pBlock->info.rows);
3776 3777

    SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
3778 3779 3780

    void* tbname = NULL;
    streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname);
3781 3782 3783 3784 3785 3786 3787
    if (tbname == NULL) {
      colDataAppendNULL(pTableCol, pBlock->info.rows);
    } else {
      char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
      STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
      colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
    }
3788
    tdbFree(tbname);
5
54liuyao 已提交
3789 3790 3791
    pBlock->info.rows += 1;
  }
  if ((*Ite) == NULL) {
5
54liuyao 已提交
3792
    tSimpleHashClear(pStDeleted);
5
54liuyao 已提交
3793 3794 3795
  }
}

5
54liuyao 已提交
3796 3797 3798 3799 3800 3801 3802 3803
static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) {
  SExprSupp*                     pSup = &pOperator->exprSupp;
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
  int32_t                        size = taosArrayGetSize(pWinArray);
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;
  int32_t                        numOfOutput = pSup->numOfExprs;
  int32_t                        numOfChildren = taosArrayGetSize(pInfo->pChildren);
3804
  ASSERT(pInfo->pChildren);
3805

3806
  for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
3807 3808 3809
    SSessionKey*      pWinKey = taosArrayGet(pWinArray, i);
    int32_t           num = 0;
    SResultWindowInfo parentWin = {0};
3810
    for (int32_t j = 0; j < numOfChildren; j++) {
X
Xiaoyu Wang 已提交
3811
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
3812
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
5
54liuyao 已提交
3813
      SStreamAggSupporter*           pChAggSup = &pChInfo->streamAggSup;
5
54liuyao 已提交
3814 3815
      SSessionKey                    chWinKey = {0};
      getSessionHashKey(pWinKey, &chWinKey);
3816 3817 3818
      SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey);
      SResultRow*      pResult = NULL;
      SResultRow*      pChResult = NULL;
5
54liuyao 已提交
3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830
      while (1) {
        SResultWindowInfo childWin = {0};
        childWin.sessionWin = *pWinKey;
        int32_t code = getSessionWinBuf(pChAggSup, pCur, &childWin);
        if (code == TSDB_CODE_SUCCESS && pWinKey->win.skey <= childWin.sessionWin.win.skey &&
            childWin.sessionWin.win.ekey <= pWinKey->win.ekey) {
          if (num == 0) {
            setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin);
            code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);
            if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
              break;
            }
3831
          }
5
54liuyao 已提交
3832 3833 3834 3835 3836 3837 3838 3839
          num++;
          updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, true);
          initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput,
                               pChild->exprSupp.rowEntryInfoOffset);
          compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
          compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL);
          saveResult(parentWin, pStUpdated);
        } else {
5
54liuyao 已提交
3840
          break;
3841 3842
        }
      }
5
54liuyao 已提交
3843 3844 3845 3846
      streamStateFreeCur(pCur);
    }
    if (num > 0) {
      saveSessionOutputBuf(pAggSup, &parentWin);
3847 3848 3849 3850
    }
  }
}

5
54liuyao 已提交
3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861
int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed) {
  void*   pIte = NULL;
  size_t  keyLen = 0;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    SResultWindowInfo* pWinInfo = pIte;
    if (isCloseWindow(&pWinInfo->sessionWin.win, pTwSup)) {
      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {
        int32_t code = saveResult(*pWinInfo, pClosed);
        if (code != TSDB_CODE_SUCCESS) {
          return code;
5
54liuyao 已提交
3862 3863
        }
      }
3864 3865
      SSessionKey* pKey = tSimpleHashGetKey(pIte, &keyLen);
      tSimpleHashIterateRemove(pHashMap, pKey, sizeof(SSessionKey), &pIte, &iter);
5
54liuyao 已提交
3866 3867 3868 3869 3870
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
3871
static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) {
5
54liuyao 已提交
3872 3873 3874 3875 3876
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);
    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
3877
    closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL);
5
54liuyao 已提交
3878 3879 3880
  }
}

5
54liuyao 已提交
3881 3882 3883 3884
int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) {
  void*   pIte = NULL;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
3885
    SResultWindowInfo* pWinInfo = pIte;
5
54liuyao 已提交
3886
    saveResult(*pWinInfo, pStUpdated);
5
54liuyao 已提交
3887 3888 3889 3890
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
3891
static void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) {
5
54liuyao 已提交
3892 3893
  int32_t size = taosArrayGetSize(pResWins);
  for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
3894 3895
    SSessionKey* pWinKey = taosArrayGet(pResWins, i);
    if (!pWinKey) continue;
5
54liuyao 已提交
3896 3897
    SSessionKey winInfo = {0};
    getSessionHashKey(pWinKey, &winInfo);
5
54liuyao 已提交
3898
    tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0);
3899 3900 3901
  }
}

5
54liuyao 已提交
3902 3903 3904 3905
void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) {
  pGroupResInfo->pRows = pArrayList;
  pGroupResInfo->index = 0;
  ASSERT(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo));
3906 3907
}

5
54liuyao 已提交
3908 3909 3910 3911 3912 3913 3914 3915 3916 3917
void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo,
                          SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  // set output datablock version
  pBlock->info.version = pTaskInfo->version;

  blockDataCleanup(pBlock);
  if (!hasRemainResults(pGroupResInfo)) {
    taosArrayDestroy(pGroupResInfo->pRows);
    pGroupResInfo->pRows = NULL;
3918 3919 3920
    return;
  }

5
54liuyao 已提交
3921 3922
  // clear the existed group id
  pBlock->info.groupId = 0;
3923
  buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
5
54liuyao 已提交
3924 3925
}

5
54liuyao 已提交
3926
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
5
54liuyao 已提交
3927
  SExprSupp*                     pSup = &pOperator->exprSupp;
5
54liuyao 已提交
3928
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3929
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
5
54liuyao 已提交
3930
  TSKEY                          maxTs = INT64_MIN;
5
54liuyao 已提交
3931
  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;
5
54liuyao 已提交
3932 3933 3934
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
3935
    doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
5
54liuyao 已提交
3936
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
3937
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
3938 3939
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
3940 3941 3942 3943
    doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
    if (pBInfo->pRes->info.rows > 0) {
      printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
      return pBInfo->pRes;
5
54liuyao 已提交
3944
    }
5
54liuyao 已提交
3945

H
Haojun Liao 已提交
3946
    setOperatorCompleted(pOperator);
5
54liuyao 已提交
3947
    return NULL;
5
54liuyao 已提交
3948 3949
  }

X
Xiaoyu Wang 已提交
3950
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
5
54liuyao 已提交
3951
  SSHashObj*     pStUpdated = tSimpleHashInit(64, hashFn);
5
54liuyao 已提交
3952
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
3953
  SArray*        pUpdated = taosArrayInit(16, sizeof(SSessionKey));  // SResKeyPos
5
54liuyao 已提交
3954 3955 3956 3957 3958
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
3959
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv");
3960

5
54liuyao 已提交
3961 3962 3963
    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
        pBlock->info.type == STREAM_CLEAR) {
      SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));
5
54liuyao 已提交
3964
      // gap must be 0
5
54liuyao 已提交
3965 3966
      doDeleteTimeWindows(pAggSup, pBlock, pWins);
      removeSessionResults(pStUpdated, pWins);
5
54liuyao 已提交
3967 3968 3969 3970 3971
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
        // gap must be 0
5
54liuyao 已提交
3972 3973
        doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL);
        rebuildSessionWindow(pOperator, pWins, pStUpdated);
5
54liuyao 已提交
3974 3975 3976 3977
      }
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
      taosArrayDestroy(pWins);
      continue;
3978
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
3979
      getAllSessionWindow(pAggSup->pResultRows, pStUpdated);
5
54liuyao 已提交
3980
      continue;
5
54liuyao 已提交
3981
    }
5
54liuyao 已提交
3982

5
54liuyao 已提交
3983 3984 3985 3986
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
3987
    // the pDataBlock are always the same one, no need to call this again
3988
    setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
3989 3990 3991 3992 3993 3994
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo));
    if (IS_FINAL_OP(pInfo)) {
      int32_t chIndex = getChildIndex(pBlock);
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
3995 3996
        SOperatorInfo* pChildOp =
            createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
5
54liuyao 已提交
3997
        if (!pChildOp) {
3998
          T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3999 4000 4001
        }
        taosArrayPush(pInfo->pChildren, &pChildOp);
      }
4002
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
4003
      setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4004
      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true);
4005
    }
5
54liuyao 已提交
4006
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
4007
    maxTs = TMAX(maxTs, pBlock->info.watermark);
5
54liuyao 已提交
4008
  }
5
54liuyao 已提交
4009 4010

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4011 4012
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
4013

5
54liuyao 已提交
4014 4015
  closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pStUpdated);
  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
5
54liuyao 已提交
4016
  copyUpdateResult(pStUpdated, pUpdated);
5
54liuyao 已提交
4017 4018 4019
  removeSessionResults(pInfo->pStDeleted, pUpdated);
  tSimpleHashCleanup(pStUpdated);
  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
5
54liuyao 已提交
4020
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
4021

4022 4023 4024 4025 4026 4027
#if 0
  char* pBuf = streamStateSessionDump(pAggSup->pState);
  qDebug("===stream===final session%s", pBuf);
  taosMemoryFree(pBuf);
#endif

4028
  doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
5
54liuyao 已提交
4029
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4030
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4031 4032
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
4033 4034 4035 4036 4037 4038 4039

  doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
  if (pBInfo->pRes->info.rows > 0) {
    printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
    return pBInfo->pRes;
  }

H
Haojun Liao 已提交
4040
  setOperatorCompleted(pOperator);
5
54liuyao 已提交
4041
  return NULL;
5
54liuyao 已提交
4042 4043
}

5
54liuyao 已提交
4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                  SExecTaskInfo* pTaskInfo) {
  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
  int32_t                        numOfCols = 0;
  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;
  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

  pOperator->pTaskInfo = pTaskInfo;

  initResultSizeInfo(&pOperator->resultInfo, 4096);
  if (pSessionNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar);
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
5
54liuyao 已提交
4064 4065
    }
  }
5
54liuyao 已提交
4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104
  SExprSupp* pSup = &pOperator->exprSupp;

  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, pSessionNode->gap,
                                pTaskInfo->streamInfo.pState, 0, 0);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pSessionNode->window.watermark,
      .calTrigger = pSessionNode->window.triggerType,
      .maxTs = INT64_MIN,
      .minTs = INT64_MAX,
  };

  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  if (pSessionNode->window.pTsEnd) {
    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;
  }
  pInfo->binfo.pRes = pResBlock;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pStDeleted = tSimpleHashInit(64, hashFn);
  pInfo->pDelIterator = NULL;
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
  pInfo->pChildren = NULL;
  pInfo->isFinal = false;
  pInfo->pPhyNode = pPhyNode;
  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;

H
Haojun Liao 已提交
4105 4106 4107 4108 4109
  setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
                  OP_NOT_OPENED, pInfo, pTaskInfo);
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL);

5
54liuyao 已提交
4110
  if (downstream) {
5
54liuyao 已提交
4111
    initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
5
54liuyao 已提交
4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128
    code = appendDownstream(pOperator, &downstream, 1);
  }
  return pOperator;

_error:
  if (pInfo != NULL) {
    destroyStreamSessionAggOperatorInfo(pInfo);
  }

  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
  tSimpleHashClear(pInfo->streamAggSup.pResultRows);
  streamStateSessionClear(pInfo->streamAggSup.pState);
5
54liuyao 已提交
4129 4130 4131 4132 4133 4134 4135
}

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
  TSKEY                          maxTs = INT64_MIN;
  SExprSupp*                     pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4136
  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;
4137

5
54liuyao 已提交
4138 4139
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
4140
  }
L
Liu Jicong 已提交
4141

4142
  {
5
54liuyao 已提交
4143
    doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
5
54liuyao 已提交
4144
    if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
4145
      printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
4146 4147 4148
      return pBInfo->pRes;
    }

4149
    doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
4150
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4151
      printDataBlock(pInfo->pDelRes, "semi session delete");
5
54liuyao 已提交
4152 4153
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
4154

4155
    if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
4156
      clearFunctionContext(&pOperator->exprSupp);
4157 4158
      // semi interval operator clear disk buffer
      clearStreamSessionOperator(pInfo);
H
Haojun Liao 已提交
4159
      setOperatorCompleted(pOperator);
4160 4161
      return NULL;
    }
5
54liuyao 已提交
4162 4163 4164
  }

  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
5
54liuyao 已提交
4165
  SSHashObj*     pStUpdated = tSimpleHashInit(64, hashFn);
5
54liuyao 已提交
4166
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
4167
  SArray*        pUpdated = taosArrayInit(16, sizeof(SSessionKey));
5
54liuyao 已提交
4168 4169 4170
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
4171
      clearSpecialDataBlock(pInfo->pUpdateRes);
4172
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
4173 4174
      break;
    }
H
Haojun Liao 已提交
4175
    printDataBlock(pBlock, "semi session recv");
5
54liuyao 已提交
4176

5
54liuyao 已提交
4177 4178
    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
        pBlock->info.type == STREAM_CLEAR) {
5
54liuyao 已提交
4179
      // gap must be 0
4180
      SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));
5
54liuyao 已提交
4181
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins);
4182
      removeSessionResults(pStUpdated, pWins);
5
54liuyao 已提交
4183
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
4184
      taosArrayDestroy(pWins);
5
54liuyao 已提交
4185
      break;
5
54liuyao 已提交
4186
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4187
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pStUpdated);
5
54liuyao 已提交
4188 4189 4190
      continue;
    }

5
54liuyao 已提交
4191 4192 4193 4194
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
5
54liuyao 已提交
4195
    // the pDataBlock are always the same one, no need to call this again
4196
    setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
4197
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, NULL, false);
5
54liuyao 已提交
4198 4199 4200 4201
    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
4202
  pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;
4203

5
54liuyao 已提交
4204
  copyUpdateResult(pStUpdated, pUpdated);
5
54liuyao 已提交
4205 4206
  removeSessionResults(pInfo->pStDeleted, pUpdated);
  tSimpleHashCleanup(pStUpdated);
5
54liuyao 已提交
4207

5
54liuyao 已提交
4208
  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
5
54liuyao 已提交
4209
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
4210

4211 4212 4213 4214 4215 4216
#if 0
  char* pBuf = streamStateSessionDump(pAggSup->pState);
  qDebug("===stream===semi session%s", pBuf);
  taosMemoryFree(pBuf);
#endif

5
54liuyao 已提交
4217
  doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
5
54liuyao 已提交
4218
  if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
4219
    printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
4220 4221 4222
    return pBInfo->pRes;
  }

4223
  doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
4224
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4225
    printDataBlock(pInfo->pDelRes, "semi session delete");
5
54liuyao 已提交
4226 4227
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
4228

5
54liuyao 已提交
4229 4230 4231
  clearFunctionContext(&pOperator->exprSupp);
  // semi interval operator clear disk buffer
  clearStreamSessionOperator(pInfo);
H
Haojun Liao 已提交
4232
  setOperatorCompleted(pOperator);
5
54liuyao 已提交
4233
  return NULL;
5
54liuyao 已提交
4234
}
4235

4236 4237
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                       SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
4238 4239
  int32_t        code = TSDB_CODE_OUT_OF_MEMORY;
  SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo);
4240 4241 4242
  if (pOperator == NULL) {
    goto _error;
  }
H
Haojun Liao 已提交
4243

4244
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
5
54liuyao 已提交
4245

H
Haojun Liao 已提交
4246
  pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION);
L
Liu Jicong 已提交
4247
  char* name = (pInfo->isFinal) ? "StreamSessionFinalAggOperator" : "StreamSessionSemiAggOperator";
H
Haojun Liao 已提交
4248 4249

  if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
H
Haojun Liao 已提交
4250
    pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
4251
    blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
H
Haojun Liao 已提交
4252
    pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL,
4253
                                           destroyStreamSessionAggOperatorInfo, NULL);
5
54liuyao 已提交
4254
  }
4255

L
Liu Jicong 已提交
4256
  setOperatorInfo(pOperator, name, pPhyNode->type, false, OP_NOT_OPENED, pInfo, pTaskInfo);
H
Haojun Liao 已提交
4257

5
54liuyao 已提交
4258 4259 4260 4261
  pOperator->operatorType = pPhyNode->type;
  if (numOfChild > 0) {
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
    for (int32_t i = 0; i < numOfChild; i++) {
5
54liuyao 已提交
4262 4263
      SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp == NULL) {
5
54liuyao 已提交
4264 4265
        goto _error;
      }
5
54liuyao 已提交
4266 4267 4268 4269
      SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
      pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
      streamStateSetNumber(pChInfo->streamAggSup.pState, i);
      taosArrayPush(pInfo->pChildren, &pChildOp);
4270 4271
    }
  }
4272 4273 4274 4275 4276

  if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }

4277 4278 4279 4280
  return pOperator;

_error:
  if (pInfo != NULL) {
4281
    destroyStreamSessionAggOperatorInfo(pInfo);
4282 4283 4284 4285 4286
  }
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5
54liuyao 已提交
4287

4288
void destroyStreamStateOperatorInfo(void* param) {
X
Xiaoyu Wang 已提交
4289
  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;
4290
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
4291
  destroyStreamAggSupporter(&pInfo->streamAggSup);
5
54liuyao 已提交
4292 4293 4294 4295
  cleanupGroupResInfo(&pInfo->groupResInfo);
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
4296 4297
      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);
      destroyOperatorInfo(pChild);
5
54liuyao 已提交
4298
    }
5
54liuyao 已提交
4299
    taosArrayDestroy(pInfo->pChildren);
5
54liuyao 已提交
4300
  }
5
54liuyao 已提交
4301 4302
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
5
54liuyao 已提交
4303
  tSimpleHashCleanup(pInfo->pSeDeleted);
D
dapan1121 已提交
4304
  taosMemoryFreeClear(param);
5
54liuyao 已提交
4305 4306 4307
}

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {
5
54liuyao 已提交
4308
  if (pWin->winInfo.sessionWin.win.skey <= ts && ts <= pWin->winInfo.sessionWin.win.ekey) {
5
54liuyao 已提交
4309 4310 4311 4312 4313 4314
    return true;
  }
  return false;
}

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {
5
54liuyao 已提交
4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346
  return pKeyData && compareVal(pKeyData, pWin->pStateKey);
}

bool compareStateKey(void* data, void* key) {
  SStateKeys* stateKey = (SStateKeys*)key;
  stateKey->pData = (char*)key + sizeof(SStateKeys);
  return compareVal(data, stateKey);
}

void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
                       SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) {
  int32_t size = pAggSup->resultRowSize;
  pCurWin->winInfo.sessionWin.groupId = groupId;
  pCurWin->winInfo.sessionWin.win.skey = ts;
  pCurWin->winInfo.sessionWin.win.ekey = ts;
  int32_t code =
      streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize,
                                    compareStateKey, &pCurWin->winInfo.pOutputBuf, &size);
  pCurWin->pStateKey =
      (SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize));
  pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
  pCurWin->pStateKey->type = pAggSup->stateKeyType;
  pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);
  pCurWin->pStateKey->isNull = false;

  if (code == TSDB_CODE_SUCCESS) {
    pCurWin->winInfo.isOutput = true;
  } else {
    if (IS_VAR_DATA_TYPE(pAggSup->stateKeyType)) {
      varDataCopy(pCurWin->pStateKey->pData, pKeyData);
    } else {
      memcpy(pCurWin->pStateKey->pData, pKeyData, pCurWin->pStateKey->bytes);
5
54liuyao 已提交
4347 4348 4349
    }
  }

5
54liuyao 已提交
4350 4351 4352 4353 4354 4355
  pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
  pNextWin->winInfo.pOutputBuf = NULL;
  SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
  code = streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0);
  if (code != TSDB_CODE_SUCCESS) {
    SET_SESSION_WIN_INVALID(pNextWin->winInfo);
5
54liuyao 已提交
4356
  }
5
54liuyao 已提交
4357
  streamStateFreeCur(pCur);
5
54liuyao 已提交
4358 4359
}

5
54liuyao 已提交
4360
int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId,
H
Haojun Liao 已提交
4361
                              SColumnInfoData* pKeyCol, int32_t rows, int32_t start, bool* allEqual,
5
54liuyao 已提交
4362
                              SSHashObj* pResultRows, SSHashObj* pSeUpdated, SSHashObj* pSeDeleted) {
5
54liuyao 已提交
4363 4364 4365 4366
  *allEqual = true;
  for (int32_t i = start; i < rows; ++i) {
    char* pKeyData = colDataGetData(pKeyCol, i);
    if (!isTsInWindow(pWinInfo, pTs[i])) {
X
Xiaoyu Wang 已提交
4367
      if (isEqualStateKey(pWinInfo, pKeyData)) {
5
54liuyao 已提交
4368
        if (IS_VALID_SESSION_WIN(pNextWin->winInfo)) {
5
54liuyao 已提交
4369
          // ts belongs to the next window
5
54liuyao 已提交
4370
          if (pTs[i] >= pNextWin->winInfo.sessionWin.win.skey) {
5
54liuyao 已提交
4371 4372 4373 4374 4375 4376 4377
            return i - start;
          }
        }
      } else {
        return i - start;
      }
    }
5
54liuyao 已提交
4378 4379

    if (pWinInfo->winInfo.sessionWin.win.skey > pTs[i]) {
H
Haojun Liao 已提交
4380
      if (pSeDeleted && pWinInfo->winInfo.isOutput) {
5
54liuyao 已提交
4381
        saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin);
5
54liuyao 已提交
4382
      }
5
54liuyao 已提交
4383 4384
      removeSessionResult(pSeUpdated, pResultRows, pWinInfo->winInfo.sessionWin);
      pWinInfo->winInfo.sessionWin.win.skey = pTs[i];
5
54liuyao 已提交
4385
    }
5
54liuyao 已提交
4386
    pWinInfo->winInfo.sessionWin.win.ekey = TMAX(pWinInfo->winInfo.sessionWin.win.ekey, pTs[i]);
5
54liuyao 已提交
4387 4388 4389 4390 4391 4392 4393
    if (!isEqualStateKey(pWinInfo, pKeyData)) {
      *allEqual = false;
    }
  }
  return rows - start;
}

5
54liuyao 已提交
4394 4395
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated,
                                 SSHashObj* pStDeleted) {
X
Xiaoyu Wang 已提交
4396
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
4397
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
4398
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
X
Xiaoyu Wang 已提交
4399 4400 4401 4402 4403
  int64_t                      groupId = pSDataBlock->info.groupId;
  int64_t                      code = TSDB_CODE_SUCCESS;
  TSKEY*                       tsCols = NULL;
  SResultRow*                  pResult = NULL;
  int32_t                      winRows = 0;
5
54liuyao 已提交
4404
  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
4405 4406
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
4407
  } else {
X
Xiaoyu Wang 已提交
4408
    return;
5
54liuyao 已提交
4409
  }
L
Liu Jicong 已提交
4410

5
54liuyao 已提交
4411
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
5
54liuyao 已提交
4412 4413
  int32_t              rows = pSDataBlock->info.rows;
  blockDataEnsureCapacity(pAggSup->pScanBlock, rows);
L
Liu Jicong 已提交
4414
  SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
5
54liuyao 已提交
4415
  for (int32_t i = 0; i < rows; i += winRows) {
5
54liuyao 已提交
4416
    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
4417 4418 4419
      i++;
      continue;
    }
5
54liuyao 已提交
4420 4421 4422 4423 4424 4425 4426 4427 4428
    char*            pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t          winIndex = 0;
    bool             allEqual = true;
    SStateWindowInfo curWin = {0};
    SStateWindowInfo nextWin = {0};
    setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin);
    setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
    winRows = updateStateWindowInfo(&curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual,
                                    pAggSup->pResultRows, pSeUpdated, pStDeleted);
5
54liuyao 已提交
4429
    if (!allEqual) {
4430
      uint64_t uid = 0;
5
54liuyao 已提交
4431 4432 4433 4434 4435
      appendOneRowToStreamSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
                                       &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
      tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
      doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);
      releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf);
5
54liuyao 已提交
4436 4437
      continue;
    }
5
54liuyao 已提交
4438 4439
    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
                              pOperator);
5
54liuyao 已提交
4440
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
4441
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4442
    }
5
54liuyao 已提交
4443 4444
    saveSessionOutputBuf(pAggSup, &curWin.winInfo);

5
54liuyao 已提交
4445
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
4446
      code = saveResult(curWin.winInfo, pSeUpdated);
5
54liuyao 已提交
4447
      if (code != TSDB_CODE_SUCCESS) {
4448
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4449 4450
      }
    }
4451 4452

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
4453 4454
      SSessionKey key = {0};
      getSessionHashKey(&curWin.winInfo.sessionWin, &key);
4455 4456
      tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
    }
5
54liuyao 已提交
4457 4458 4459 4460 4461 4462 4463 4464
  }
}

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

4465
  SExprSupp*                   pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4466
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4467
  SOptrBasicInfo*              pBInfo = &pInfo->binfo;
L
Liu Jicong 已提交
4468
  int64_t                      maxTs = INT64_MIN;
5
54liuyao 已提交
4469
  if (pOperator->status == OP_RES_TO_RETURN) {
4470
    doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
5
54liuyao 已提交
4471
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4472
      printDataBlock(pInfo->pDelRes, "single state delete");
5
54liuyao 已提交
4473 4474
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
4475 4476 4477 4478 4479

    doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
    if (pBInfo->pRes->info.rows > 0) {
      printDataBlock(pBInfo->pRes, "single state");
      return pBInfo->pRes;
5
54liuyao 已提交
4480
    }
5
54liuyao 已提交
4481

H
Haojun Liao 已提交
4482
    setOperatorCompleted(pOperator);
5
54liuyao 已提交
4483
    return NULL;
5
54liuyao 已提交
4484 4485
  }

X
Xiaoyu Wang 已提交
4486
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
5
54liuyao 已提交
4487
  SSHashObj*     pSeUpdated = tSimpleHashInit(64, hashFn);
5
54liuyao 已提交
4488
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
4489
  SArray*        pUpdated = taosArrayInit(16, sizeof(SSessionKey));
5
54liuyao 已提交
4490 4491 4492 4493 4494
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4495
    printDataBlock(pBlock, "single state recv");
4496

5
54liuyao 已提交
4497 4498 4499 4500
    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
        pBlock->info.type == STREAM_CLEAR) {
      SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins);
4501
      removeSessionResults(pSeUpdated, pWins);
5
54liuyao 已提交
4502
      copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
4503 4504
      taosArrayDestroy(pWins);
      continue;
4505
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4506
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pSeUpdated);
5
54liuyao 已提交
4507
      continue;
5
54liuyao 已提交
4508
    }
4509

5
54liuyao 已提交
4510 4511 4512 4513
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4514
    // the pDataBlock are always the same one, no need to call this again
4515
    setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4516
    doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
4517
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
4518
  }
4519
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4520 4521
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
X
Xiaoyu Wang 已提交
4522

5
54liuyao 已提交
4523
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pSeUpdated);
5
54liuyao 已提交
4524
  copyUpdateResult(pSeUpdated, pUpdated);
5
54liuyao 已提交
4525 4526
  removeSessionResults(pInfo->pSeDeleted, pUpdated);
  tSimpleHashCleanup(pSeUpdated);
5
54liuyao 已提交
4527

5
54liuyao 已提交
4528
  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
5
54liuyao 已提交
4529
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
4530

5
54liuyao 已提交
4531 4532 4533 4534 4535 4536
#if 0
  char* pBuf = streamStateSessionDump(pInfo->streamAggSup.pState);
  qDebug("===stream===final session%s", pBuf);
  taosMemoryFree(pBuf);
#endif

4537
  doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
5
54liuyao 已提交
4538
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4539
    printDataBlock(pInfo->pDelRes, "single state delete");
5
54liuyao 已提交
4540 4541 4542
    return pInfo->pDelRes;
  }

5
54liuyao 已提交
4543 4544 4545 4546 4547
  doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
  if (pBInfo->pRes->info.rows > 0) {
    printDataBlock(pBInfo->pRes, "single state");
    return pBInfo->pRes;
  }
H
Haojun Liao 已提交
4548
  setOperatorCompleted(pOperator);
5
54liuyao 已提交
4549
  return NULL;
4550 4551
}

X
Xiaoyu Wang 已提交
4552 4553 4554 4555 4556
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
H
Haojun Liao 已提交
4557
  int32_t                      code = TSDB_CODE_SUCCESS;
5
54liuyao 已提交
4558

X
Xiaoyu Wang 已提交
4559 4560
  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
4561
  if (pInfo == NULL || pOperator == NULL) {
H
Haojun Liao 已提交
4562
    code = TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
4563 4564 4565 4566
    goto _error;
  }

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
4567
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
4568 4569 4570
  if (pStateNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
H
Haojun Liao 已提交
4571
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
5
54liuyao 已提交
4572 4573 4574 4575 4576
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

X
Xiaoyu Wang 已提交
4577 4578
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
5
54liuyao 已提交
4579 4580
      .calTrigger = pStateNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
4581
      .minTs = INT64_MAX,
X
Xiaoyu Wang 已提交
4582
  };
4583

5
54liuyao 已提交
4584
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
4585

5
54liuyao 已提交
4586 4587 4588
  SExprSupp*   pSup = &pOperator->exprSupp;
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);
H
Haojun Liao 已提交
4589
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
4590
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
4591 4592 4593
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
5
54liuyao 已提交
4594 4595 4596 4597
  int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes;
  int16_t type = pColNode->node.resType.type;
  code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize,
                                type);
5
54liuyao 已提交
4598 4599 4600 4601 4602 4603
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->primaryTsIndex = tsSlotId;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
5
54liuyao 已提交
4604
  pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);
5
54liuyao 已提交
4605
  pInfo->pDelIterator = NULL;
H
Haojun Liao 已提交
4606
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
5
54liuyao 已提交
4607
  pInfo->pChildren = NULL;
5
54liuyao 已提交
4608
  pInfo->ignoreExpiredData = pStateNode->window.igExpired;
5
54liuyao 已提交
4609

L
Liu Jicong 已提交
4610 4611
  setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
                  pInfo, pTaskInfo);
4612
  pOperator->fpSet =
H
Haojun Liao 已提交
4613
      createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL);
5
54liuyao 已提交
4614
  initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
5
54liuyao 已提交
4615 4616 4617 4618 4619 4620 4621
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
  return pOperator;

_error:
4622
  destroyStreamStateOperatorInfo(pInfo);
5
54liuyao 已提交
4623 4624 4625 4626
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
4627

4628
void destroyMAIOperatorInfo(void* param) {
4629
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param;
4630
  destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo);
D
dapan1121 已提交
4631
  taosMemoryFreeClear(param);
4632 4633
}

4634
static SResultRow* doSetSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, SAggSupporter* pSup) {
H
Haojun Liao 已提交
4635 4636
  SResultRow* pResult = getNewResultRow(pSup->pResultBuf, &pSup->currentPageId, pSup->resultRowSize);
  pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
4637 4638
  return pResult;
}
4639

4640 4641 4642 4643 4644 4645 4646 4647
static int32_t setSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, SResultRow** pResult,
                                       SExprSupp* pExprSup, SAggSupporter* pAggSup) {
  if (*pResult == NULL) {
    *pResult = doSetSingleOutputTupleBuf(pResultRowInfo, pAggSup);
    if (*pResult == NULL) {
      return terrno;
    }
  }
4648

4649
  // set time window for current result
4650 4651
  (*pResult)->win = (*win);
  setResultRowInitCtx((*pResult), pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
4652
  return TSDB_CODE_SUCCESS;
4653 4654
}

4655
static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
4656
                                          SSDataBlock* pBlock, SSDataBlock* pResultBlock) {
4657
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
D
dapan1121 已提交
4658
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
4659 4660

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
4661
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
4662
  SInterval*     pInterval = &iaInfo->interval;
4663

5
54liuyao 已提交
4664 4665
  int32_t  startPos = 0;
  int64_t* tsCols = extractTsCol(pBlock, iaInfo);
4666

4667 4668
  TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols);

4669 4670
  // there is an result exists
  if (miaInfo->curTs != INT64_MIN) {
4671
    if (ts != miaInfo->curTs) {
4672
      finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo);
4673
      resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow));
4674
      miaInfo->curTs = ts;
4675
    }
4676 4677
  } else {
    miaInfo->curTs = ts;
4678 4679 4680
  }

  STimeWindow win = {0};
4681
  win.skey = miaInfo->curTs;
4682
  win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
4683

5
54liuyao 已提交
4684
  int32_t ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
4685 4686
  if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) {
    T_LONG_JMP(pTaskInfo->env, ret);
4687 4688
  }

4689 4690
  int32_t currPos = startPos;

4691
  STimeWindow currWin = win;
4692
  while (++currPos < pBlock->info.rows) {
4693
    if (tsCols[currPos] == miaInfo->curTs) {
4694
      continue;
4695 4696 4697
    }

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
4698
    doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
4699
                     pBlock->info.rows, pSup->numOfExprs);
4700

4701
    finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo);
4702
    resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow));
4703
    miaInfo->curTs = tsCols[currPos];
4704

4705
    currWin.skey = miaInfo->curTs;
4706
    currWin.ekey = taosTimeAdd(currWin.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
4707 4708

    startPos = currPos;
5
54liuyao 已提交
4709
    ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
4710 4711
    if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) {
      T_LONG_JMP(pTaskInfo->env, ret);
4712
    }
4713 4714

    miaInfo->curTs = currWin.skey;
4715
  }
4716

4717
  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
4718
  doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
4719
                   pBlock->info.rows, pSup->numOfExprs);
4720 4721
}

4722 4723 4724 4725
static void cleanupAfterGroupResultGen(SMergeAlignedIntervalAggOperatorInfo* pMiaInfo, SSDataBlock* pRes) {
  pRes->info.groupId = pMiaInfo->groupId;
  pMiaInfo->curTs = INT64_MIN;
  pMiaInfo->groupId = 0;
4726 4727
}

4728
static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
S
shenglian zhou 已提交
4729
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
4730

4731 4732
  SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info;
  SIntervalAggOperatorInfo*             pIaInfo = pMiaInfo->intervalAggOperatorInfo;
4733

4734 4735 4736 4737 4738
  SExprSupp*      pSup = &pOperator->exprSupp;
  SSDataBlock*    pRes = pIaInfo->binfo.pRes;
  SResultRowInfo* pResultRowInfo = &pIaInfo->binfo.resultRowInfo;
  SOperatorInfo*  downstream = pOperator->pDownstream[0];
  int32_t         scanFlag = MAIN_SCAN;
4739

4740 4741
  while (1) {
    SSDataBlock* pBlock = NULL;
4742
    if (pMiaInfo->prefetchedBlock == NULL) {
4743 4744
      pBlock = downstream->fpSet.getNextFn(downstream);
    } else {
4745 4746
      pBlock = pMiaInfo->prefetchedBlock;
      pMiaInfo->prefetchedBlock = NULL;
4747

4748
      pMiaInfo->groupId = pBlock->info.groupId;
4749
    }
4750

4751
    // no data exists, all query processing is done
4752
    if (pBlock == NULL) {
4753 4754 4755
      // close last unclosed time window
      if (pMiaInfo->curTs != INT64_MIN) {
        finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo);
4756 4757
        resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow));
        cleanupAfterGroupResultGen(pMiaInfo, pRes);
4758
      }
4759

H
Haojun Liao 已提交
4760
      setOperatorCompleted(pOperator);
4761
      break;
4762
    }
4763

H
Haojun Liao 已提交
4764 4765 4766
    if (pMiaInfo->groupId == 0) {
      if (pMiaInfo->groupId != pBlock->info.groupId) {
        pMiaInfo->groupId = pBlock->info.groupId;
5
54liuyao 已提交
4767
        pRes->info.groupId = pMiaInfo->groupId;
H
Haojun Liao 已提交
4768 4769 4770 4771 4772 4773
      }
    } else {
      if (pMiaInfo->groupId != pBlock->info.groupId) {
        // if there are unclosed time window, close it firstly.
        ASSERT(pMiaInfo->curTs != INT64_MIN);
        finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo);
4774
        resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow));
H
Haojun Liao 已提交
4775

4776 4777
        pMiaInfo->prefetchedBlock = pBlock;
        cleanupAfterGroupResultGen(pMiaInfo, pRes);
H
Haojun Liao 已提交
4778
        break;
5
54liuyao 已提交
4779
      } else {
H
Haojun Liao 已提交
4780
        // continue
5
54liuyao 已提交
4781
        pRes->info.groupId = pMiaInfo->groupId;
H
Haojun Liao 已提交
4782
      }
4783
    }
4784

4785
    getTableScanInfo(pOperator, &pIaInfo->inputOrder, &scanFlag);
4786
    setInputDataBlock(pSup, pBlock, pIaInfo->inputOrder, scanFlag, true);
4787
    doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes);
4788

H
Haojun Liao 已提交
4789
    doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
4790 4791 4792
    if (pRes->info.rows >= pOperator->resultInfo.capacity) {
      break;
    }
4793
  }
4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808
}

static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info;
  SIntervalAggOperatorInfo*             iaInfo = pMiaInfo->intervalAggOperatorInfo;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);

  if (iaInfo->binfo.mergeResultBlock) {
dengyihao's avatar
dengyihao 已提交
4809
    while (1) {
4810
      if (pOperator->status == OP_EXEC_DONE) {
4811 4812
        break;
      }
4813

4814
      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
4815 4816 4817
        break;
      }

4818 4819 4820 4821
      doMergeAlignedIntervalAgg(pOperator);
    }
  } else {
    doMergeAlignedIntervalAgg(pOperator);
4822 4823 4824 4825 4826 4827 4828
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

4829
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode,
4830
                                                      SExecTaskInfo* pTaskInfo) {
4831
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo));
4832
  SOperatorInfo*                        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
4833 4834 4835 4836
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

D
dapan1121 已提交
4837 4838 4839 4840 4841
  miaInfo->intervalAggOperatorInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
  if (miaInfo->intervalAggOperatorInfo == NULL) {
    goto _error;
  }

4842 4843 4844 4845 4846 4847 4848
  SInterval interval = {.interval = pNode->interval,
                        .sliding = pNode->sliding,
                        .intervalUnit = pNode->intervalUnit,
                        .slidingUnit = pNode->slidingUnit,
                        .offset = pNode->offset,
                        .precision = ((SColumnNode*)pNode->window.pTspk)->node.resType.precision};

D
dapan1121 已提交
4849
  SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
4850
  SExprSupp*                pSup = &pOperator->exprSupp;
4851

H
Haojun Liao 已提交
4852 4853 4854 4855 4856
  int32_t code = filterInitFromNode((SNode*)pNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

L
Liu Jicong 已提交
4857 4858 4859 4860 4861
  miaInfo->curTs = INT64_MIN;
  iaInfo->win = pTaskInfo->window;
  iaInfo->inputOrder = TSDB_ORDER_ASC;
  iaInfo->interval = interval;
  iaInfo->execModel = pTaskInfo->execModel;
4862 4863
  iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId;
  iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock;
4864 4865

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
4866
  initResultSizeInfo(&pOperator->resultInfo, 4096);
4867

H
Haojun Liao 已提交
4868 4869
  int32_t    num = 0;
  SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
H
Haojun Liao 已提交
4870 4871

  code = initAggInfo(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
4872 4873 4874
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
4875

H
Haojun Liao 已提交
4876
  SSDataBlock* pResBlock = createResDataBlock(pNode->window.node.pOutputDataBlockDesc);
4877
  initBasicInfo(&iaInfo->binfo, pResBlock);
4878
  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
4879

4880
  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, iaInfo);
4881
  if (iaInfo->timeWindowInterpo) {
4882
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
4883 4884
  }

4885
  initResultRowInfo(&iaInfo->binfo.resultRowInfo);
4886
  blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity);
L
Liu Jicong 已提交
4887 4888
  setOperatorInfo(pOperator, "TimeMergeAlignedIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL,
                  false, OP_NOT_OPENED, miaInfo, pTaskInfo);
4889

4890
  pOperator->fpSet =
H
Haojun Liao 已提交
4891
      createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, NULL);
4892 4893 4894 4895 4896 4897 4898 4899 4900

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
4901
  destroyMAIOperatorInfo(miaInfo);
4902 4903 4904 4905
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
4906 4907 4908 4909 4910

//=====================================================================================================================
// merge interval operator
typedef struct SMergeIntervalAggOperatorInfo {
  SIntervalAggOperatorInfo intervalAggOperatorInfo;
L
Liu Jicong 已提交
4911 4912 4913 4914 4915 4916
  SList*                   groupIntervals;
  SListIter                groupIntervalsIter;
  bool                     hasGroupId;
  uint64_t                 groupId;
  SSDataBlock*             prefetchedBlock;
  bool                     inputBlocksFinished;
4917 4918
} SMergeIntervalAggOperatorInfo;

S
slzhou 已提交
4919
typedef struct SGroupTimeWindow {
L
Liu Jicong 已提交
4920
  uint64_t    groupId;
S
slzhou 已提交
4921 4922 4923
  STimeWindow window;
} SGroupTimeWindow;

4924
void destroyMergeIntervalOperatorInfo(void* param) {
4925
  SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
S
slzhou 已提交
4926
  tdListFree(miaInfo->groupIntervals);
4927
  destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo);
4928

D
dapan1121 已提交
4929
  taosMemoryFreeClear(param);
4930 4931
}

L
Liu Jicong 已提交
4932 4933
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win,
                                    SSDataBlock* pResultBlock) {
4934 4935 4936
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
4937
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
4938 4939 4940
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId);
L
Liu Jicong 已提交
4941 4942
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(
      iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
4943
  ASSERT(p1 != NULL);
5
54liuyao 已提交
4944
  //  finalizeResultRows(iaInfo->aggSup.pResultBuf, p1, pResultBlock, pTaskInfo);
4945
  tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
4946 4947 4948
  return TSDB_CODE_SUCCESS;
}

4949 4950 4951 4952
static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock,
                                        STimeWindow* newWin) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
4953
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
4954

S
slzhou 已提交
4955 4956
  SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin};
  tdListAppend(miaInfo->groupIntervals, &groupTimeWindow);
4957

S
slzhou 已提交
4958 4959 4960 4961 4962
  SListIter iter = {0};
  tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD);
  SListNode* listNode = NULL;
  while ((listNode = tdListNext(&iter)) != NULL) {
    SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
L
Liu Jicong 已提交
4963
    if (prevGrpWin->groupId != tableGroupId) {
S
slzhou 已提交
4964 4965
      continue;
    }
4966

S
slzhou 已提交
4967
    STimeWindow* prevWin = &prevGrpWin->window;
H
Haojun Liao 已提交
4968
    if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) {
5
54liuyao 已提交
4969
      //      finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
S
slzhou 已提交
4970 4971
      tdListPopNode(miaInfo->groupIntervals, listNode);
    }
4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988
  }

  return 0;
}

static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                                   int32_t scanFlag, SSDataBlock* pResultBlock) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*     pExprSup = &pOperatorInfo->exprSupp;

  int32_t     startPos = 0;
  int32_t     numOfOutput = pExprSup->numOfExprs;
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
4989
  bool        ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
4990 4991 4992
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

4993 4994
  STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval,
                                        iaInfo->inputOrder);
4995 4996 4997 4998 4999

  int32_t ret =
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                             numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
5000
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5001 5002 5003 5004
  }

  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
5005
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5006 5007 5008 5009
  ASSERT(forwardRows > 0);

  // prev time window not interpolation yet.
  if (iaInfo->timeWindowInterpo) {
5010
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
5011 5012 5013 5014 5015 5016
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);

    // restore current time window
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                                 numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS) {
5017
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5018 5019 5020 5021 5022 5023 5024
    }

    // window start key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup);
  }

  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true);
5025 5026
  doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
                   pBlock->info.rows, numOfOutput);
5027 5028 5029 5030 5031 5032 5033 5034
  doCloseWindow(pResultRowInfo, iaInfo, pResult);

  // output previous interval results after this interval (&win) is closed
  outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win);

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = forwardRows - 1 + startPos;
5035 5036
    startPos =
        getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->inputOrder);
5037 5038 5039 5040 5041 5042 5043 5044 5045
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
    int32_t code =
        setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                               pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
5046
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5047 5048 5049 5050
    }

    ekey = ascScan ? nextWin.ekey : nextWin.skey;
    forwardRows =
5051
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5052 5053 5054 5055 5056

    // window start(end) key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup);

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true);
5057 5058
    doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pBlock->info.rows, numOfOutput);
5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098
    doCloseWindow(pResultRowInfo, iaInfo, pResult);

    // output previous interval results after this interval (&nextWin) is closed
    outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin);
  }

  if (iaInfo->timeWindowInterpo) {
    saveDataBlockLastRow(iaInfo->pPrevValues, pBlock, iaInfo->pInterpCols);
  }
}

static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExprSupp*                     pExpSupp = &pOperator->exprSupp;

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);

  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
        miaInfo->groupId = pBlock->info.groupId;
        miaInfo->prefetchedBlock = NULL;
      }

      if (pBlock == NULL) {
S
slzhou 已提交
5099
        tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD);
5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111
        miaInfo->inputBlocksFinished = true;
        break;
      }

      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
        miaInfo->groupId = pBlock->info.groupId;
      } else if (miaInfo->groupId != pBlock->info.groupId) {
        miaInfo->prefetchedBlock = pBlock;
        break;
      }

5112
      getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag);
5113
      setInputDataBlock(pExpSupp, pBlock, iaInfo->inputOrder, scanFlag, true);
5114 5115 5116 5117 5118 5119 5120 5121
      doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);

      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
        break;
      }
    }

    pRes->info.groupId = miaInfo->groupId;
5122 5123 5124
  }

  if (miaInfo->inputBlocksFinished) {
S
slzhou 已提交
5125
    SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter);
5126

S
slzhou 已提交
5127 5128
    if (listNode != NULL) {
      SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data);
5
54liuyao 已提交
5129
      //      finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
S
slzhou 已提交
5130
      pRes->info.groupId = grpWin->groupId;
5131 5132 5133 5134
    }
  }

  if (pRes->info.rows == 0) {
H
Haojun Liao 已提交
5135
    setOperatorCompleted(pOperator);
5136 5137 5138 5139 5140 5141 5142
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

5143 5144 5145
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode,
                                               SExecTaskInfo* pTaskInfo) {
  SMergeIntervalAggOperatorInfo* pMergeIntervalInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
5146
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5147
  if (pMergeIntervalInfo == NULL || pOperator == NULL) {
5148 5149 5150
    goto _error;
  }

5
54liuyao 已提交
5151 5152
  int32_t    num = 0;
  SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
5153 5154 5155 5156 5157 5158 5159

  SInterval interval = {.interval = pIntervalPhyNode->interval,
                        .sliding = pIntervalPhyNode->sliding,
                        .intervalUnit = pIntervalPhyNode->intervalUnit,
                        .slidingUnit = pIntervalPhyNode->slidingUnit,
                        .offset = pIntervalPhyNode->offset,
                        .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
5160

5161
  pMergeIntervalInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow));
5162

5163
  SIntervalAggOperatorInfo* pIntervalInfo = &pMergeIntervalInfo->intervalAggOperatorInfo;
L
Liu Jicong 已提交
5164
  pIntervalInfo->win = pTaskInfo->window;
5165
  pIntervalInfo->inputOrder = TSDB_ORDER_ASC;
L
Liu Jicong 已提交
5166 5167
  pIntervalInfo->interval = interval;
  pIntervalInfo->execModel = pTaskInfo->execModel;
5168 5169
  pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock;
  pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
5170 5171 5172 5173

  SExprSupp* pExprSupp = &pOperator->exprSupp;

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
5174
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5175

5176
  int32_t code = initAggInfo(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
5177 5178 5179
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
5180

H
Haojun Liao 已提交
5181
  SSDataBlock* pResBlock = createResDataBlock(pIntervalPhyNode->window.node.pOutputDataBlockDesc);
5182 5183
  initBasicInfo(&pIntervalInfo->binfo, pResBlock);
  initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win);
5184

5185 5186
  pIntervalInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, num, pIntervalInfo);
  if (pIntervalInfo->timeWindowInterpo) {
5187
    pIntervalInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
5188
    if (pIntervalInfo->binfo.resultRowInfo.openWindow == NULL) {
5189 5190 5191 5192
      goto _error;
    }
  }

5193
  initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo);
L
Liu Jicong 已提交
5194 5195
  setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false,
                  OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo);
5196
  pOperator->fpSet =
H
Haojun Liao 已提交
5197
      createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, NULL);
5198 5199 5200 5201 5202 5203 5204 5205 5206

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
H
Haojun Liao 已提交
5207 5208 5209 5210
  if (pMergeIntervalInfo != NULL) {
    destroyMergeIntervalOperatorInfo(pMergeIntervalInfo);
  }

5211 5212 5213
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
5214
}
5215 5216 5217

static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
5218 5219
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
  int64_t                      maxTs = INT64_MIN;
5
54liuyao 已提交
5220
  int64_t                      minTs = INT64_MAX;
5221
  SExprSupp*                   pSup = &pOperator->exprSupp;
5222 5223 5224 5225 5226 5227

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
5228
    doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
5229
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
5230
      printDataBlock(pInfo->pDelRes, "single interval delete");
5231 5232 5233
      return pInfo->pDelRes;
    }

5234
    doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
5235 5236 5237
    if (pInfo->binfo.pRes->info.rows > 0) {
      printDataBlock(pInfo->binfo.pRes, "single interval");
      return pInfo->binfo.pRes;
5238
    }
5239 5240
    deleteIntervalDiscBuf(pInfo->pState, NULL, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, &pInfo->interval,
                          &pInfo->delKey);
H
Haojun Liao 已提交
5241
    setOperatorCompleted(pOperator);
L
Liu Jicong 已提交
5242
    streamStateCommit(pTaskInfo->streamInfo.pState);
5
54liuyao 已提交
5243
    return NULL;
5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];

  SArray*    pUpdated = taosArrayInit(4, POINTER_BYTES);  // SResKeyPos
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*  pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
    printDataBlock(pBlock, "single interval recv");

5259 5260 5261
    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
        pBlock->info.type == STREAM_CLEAR) {
      doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pUpdatedMap);
5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280
      continue;
    } else if (pBlock->info.type == STREAM_GET_ALL) {
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
      continue;
    }

    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {
      // set input version
      pTaskInfo->version = pBlock->info.version;
    }

    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }

    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
    // caller. Note that all the time window are not close till now.
    // the pDataBlock are always the same one, no need to call this again
5281
    setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5282 5283 5284 5285 5286
    if (pInfo->invertible) {
      setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type);
    }

    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
5287
    minTs = TMIN(minTs, pBlock->info.window.skey);
H
Haojun Liao 已提交
5288

5289
    doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
5290 5291
  }
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
5292
  pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs);
5293
  pOperator->status = OP_RES_TO_RETURN;
5294
  removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
5
54liuyao 已提交
5295
  closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap,
5296
                            pInfo->pDelWins, pOperator);
5297 5298 5299 5300 5301 5302 5303 5304 5305 5306

  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosArraySort(pUpdated, resultrowComparAsc);

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  taosHashCleanup(pUpdatedMap);
5
54liuyao 已提交
5307

5308
  doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
5309
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
5310
    printDataBlock(pInfo->pDelRes, "single interval delete");
5311 5312 5313
    return pInfo->pDelRes;
  }

5314
  doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
5315 5316 5317 5318 5319 5320
  if (pInfo->binfo.pRes->info.rows > 0) {
    printDataBlock(pInfo->binfo.pRes, "single interval");
    return pInfo->binfo.pRes;
  }

  return NULL;
5321 5322 5323 5324 5325
}

SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
5326
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5327 5328 5329 5330 5331
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

H
Haojun Liao 已提交
5332
  int32_t    code = TSDB_CODE_SUCCESS;
5333 5334
  int32_t    numOfCols = 0;
  SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5335
  ASSERT(numOfCols > 0);
H
Haojun Liao 已提交
5336

5337
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
5338 5339 5340 5341 5342 5343 5344 5345
  SInterval    interval = {
         .interval = pIntervalPhyNode->interval,
         .sliding = pIntervalPhyNode->sliding,
         .intervalUnit = pIntervalPhyNode->intervalUnit,
         .slidingUnit = pIntervalPhyNode->slidingUnit,
         .offset = pIntervalPhyNode->offset,
         .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,
  };
H
Haojun Liao 已提交
5346

5347 5348 5349 5350
  STimeWindowAggSupp twAggSupp = {
      .waterMark = pIntervalPhyNode->window.watermark,
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
5351
      .minTs = INT64_MAX,
5
54liuyao 已提交
5352
      .deleteMark = INT64_MAX,
5353
  };
H
Haojun Liao 已提交
5354

5355
  ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY);
5356

5357 5358 5359 5360 5361 5362 5363
  pOperator->pTaskInfo = pTaskInfo;
  pInfo->interval = interval;
  pInfo->twAggSup = twAggSupp;
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
  pInfo->isFinal = false;

  SExprSupp* pSup = &pOperator->exprSupp;
H
Haojun Liao 已提交
5364 5365 5366 5367
  initBasicInfo(&pInfo->binfo, pResBlock);
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

5368
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
5369
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5370

5371
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
H
Haojun Liao 已提交
5372
  code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
5373 5374 5375 5376
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
5377 5378 5379 5380 5381 5382 5383 5384
  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
5385 5386

  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
5387
  pInfo->invertible = false;
5388 5389 5390 5391 5392
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
  pInfo->delIndex = 0;
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
  initResultRowInfo(&pInfo->binfo.resultRowInfo);

5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406
  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
  *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
  streamStateSetNumber(pInfo->pState, -1);

  pInfo->pPhyNode = NULL;  // create new child
  pInfo->pPullDataMap = NULL;
  pInfo->pPullWins = NULL;  // SPullWindowInfo
  pInfo->pullIndex = 0;
  pInfo->pPullDataRes = NULL;
  pInfo->isFinal = false;
  pInfo->pChildren = NULL;
  pInfo->delKey.ts = INT64_MAX;
  pInfo->delKey.groupId = 0;

L
Liu Jicong 已提交
5407 5408 5409 5410
  setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,
                  pInfo, pTaskInfo);
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL);
5411

5
54liuyao 已提交
5412
  initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
5413 5414 5415 5416 5417 5418 5419 5420
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5421
  destroyStreamFinalIntervalOperatorInfo(pInfo);
5422 5423 5424 5425
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}