timewindowoperator.c 191.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15
#include "executorimpl.h"
16
#include "filter.h"
X
Xiaoyu Wang 已提交
17
#include "function.h"
5
54liuyao 已提交
18
#include "functionMgt.h"
H
Haojun Liao 已提交
19
#include "tcommon.h"
5
54liuyao 已提交
20
#include "tcompare.h"
L
Liu Jicong 已提交
21
#include "tdatablock.h"
H
Haojun Liao 已提交
22
#include "tfill.h"
dengyihao's avatar
dengyihao 已提交
23
#include "tglobal.h"
dengyihao's avatar
dengyihao 已提交
24
#include "tlog.h"
25
#include "ttime.h"
26

L
Liu Jicong 已提交
27
#define IS_FINAL_OP(op)    ((op)->isFinal)
5
54liuyao 已提交
28
#define DEAULT_DELETE_MARK (1000LL * 60LL * 60LL * 24LL * 365LL * 10LL);
H
Haojun Liao 已提交
29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53

typedef struct SSessionAggOperatorInfo {
  SOptrBasicInfo     binfo;
  SAggSupporter      aggSup;
  SGroupResInfo      groupResInfo;
  SWindowRowsSup     winSup;
  bool               reptScan;  // next round scan
  int64_t            gap;       // session window gap
  int32_t            tsSlotId;  // primary timestamp slot id
  STimeWindowAggSupp twAggSup;
} SSessionAggOperatorInfo;

typedef struct SStateWindowOperatorInfo {
  SOptrBasicInfo     binfo;
  SAggSupporter      aggSup;
  SExprSupp          scalarSup;
  SGroupResInfo      groupResInfo;
  SWindowRowsSup     winSup;
  SColumn            stateCol;  // start row index
  bool               hasKey;
  SStateKeys         stateKey;
  int32_t            tsSlotId;  // primary timestamp column slot id
  STimeWindowAggSupp twAggSup;
} SStateWindowOperatorInfo;

54 55 56 57 58
typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;

5
54liuyao 已提交
59 60
typedef struct SPullWindowInfo {
  STimeWindow window;
L
Liu Jicong 已提交
61
  uint64_t    groupId;
5
54liuyao 已提交
62
  STimeWindow calWin;
5
54liuyao 已提交
63 64
} SPullWindowInfo;

65 66
typedef struct SOpenWindowInfo {
  SResultRowPosition pos;
L
Liu Jicong 已提交
67
  uint64_t           groupId;
68 69
} SOpenWindowInfo;

70 71
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);

L
Liu Jicong 已提交
72 73
static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult,
                                              uint64_t groupId);
74 75
static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult);

X
Xiaoyu Wang 已提交
76
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }
77 78 79

static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
80
                                      int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
81 82
                                      SExecTaskInfo* pTaskInfo) {
  SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
D
dapan1121 已提交
83
                                                  masterscan, tableGroupId, pTaskInfo, true, pAggSup, true);
84 85 86 87 88 89 90 91

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // set time window for current result
  pResultRow->win = (*win);
92

93
  *pResult = pResultRow;
94
  setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
95

96 97 98 99 100 101 102 103 104 105 106 107 108
  return TSDB_CODE_SUCCESS;
}

static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) {
  int64_t* ts = (int64_t*)pColData->pData;
  int32_t  delta = includeEndpoint ? 1 : 0;

  int64_t duration = pWin->ekey - pWin->skey + delta;
  ts[2] = duration;            // set the duration
  ts[3] = pWin->skey;          // window start key
  ts[4] = pWin->ekey + delta;  // window end key
}

109
static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) {
110 111 112
  pRowSup->win.ekey = ts;
  pRowSup->prevTs = ts;
  pRowSup->numOfRows += 1;
113
  pRowSup->groupId = groupId;
114 115
}

dengyihao's avatar
dengyihao 已提交
116 117
static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex,
                                     uint64_t groupId) {
118 119 120
  pRowSup->startRowIndex = rowIndex;
  pRowSup->numOfRows = 0;
  pRowSup->win.skey = tsList[rowIndex];
121
  pRowSup->groupId = groupId;
122 123
}

L
Liu Jicong 已提交
124 125
FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos,
                                            int32_t order, int64_t* pData) {
126
  int32_t forwardRows = 0;
127 128 129 130

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
131
      forwardRows = end;
132

S
slzhou 已提交
133
      while (pData[end + pos] == ekey) {
134
        forwardRows += 1;
S
slzhou 已提交
135
        ++pos;
136 137 138
      }
    }
  } else {
139
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
140
    if (end >= 0) {
141
      forwardRows = end;
142

S
slzhou 已提交
143
      while (pData[end + pos] == ekey) {
144
        forwardRows += 1;
S
slzhou 已提交
145
        ++pos;
146 147
      }
    }
X
Xiaoyu Wang 已提交
148 149 150 151 152 153 154 155
    //    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
    //    if (end >= 0) {
    //      forwardRows = pos - end;
    //
    //      if (pData[end] == ekey) {
    //        forwardRows += 1;
    //      }
    //    }
156 157
  }

158 159
  assert(forwardRows >= 0);
  return forwardRows;
160 161
}

5
54liuyao 已提交
162
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY*  keyList = (TSKEY*)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
179 180 181 182 183 184 185 186 187 188 189
      if (key >= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key < keyList[lastPos]) {
        lastPos += 1;
        if (lastPos >= num) {
          return -1;
        } else {
          return lastPos;
        }
      }
190 191 192 193 194 195

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        firstPos = midPos + 1;
196 197
      } else if (key > keyList[midPos]) {
        lastPos = midPos - 1;
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

X
Xiaoyu Wang 已提交
233 234
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
                                 __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num = -1;
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);

  if (order == TSDB_ORDER_ASC) {
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
      if (item != NULL) {
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
256
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
257 258
      }
    } else {
259
      num = pDataBlockInfo->rows - startPos;
260
      if (item != NULL) {
261
        item->lastKey = pDataBlockInfo->window.ekey + step;
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
      }
    }
  }

  assert(num >= 0);
  return num;
}

static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
    return;
  }

  int64_t key = tw->skey, interval = pInterval->interval;
  // convert key to second
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;

  if (pInterval->intervalUnit == 'y') {
    interval *= 12;
  }

  struct tm tm;
  time_t    t = (time_t)key;
288
  taosLocalTime(&t, &tm, NULL);
289 290 291 292

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
293
  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
294 295 296 297

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
298
  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
299 300 301 302

  tw->ekey -= 1;
}

5
54liuyao 已提交
303 304 305 306
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) {
  getNextTimeWindow(pInterval, pInterval->precision, order, tw);
}

307 308
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs,
                               int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
309
  SqlFunctionCtx* pCtx = pSup->pCtx;
310

311
  int32_t index = 1;
312
  for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
H
Haojun Liao 已提交
313
    if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
314 315 316 317
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

X
Xiaoyu Wang 已提交
318
    SFunctParam*     pParam = &pCtx[k].param[0];
319 320
    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId);

321
    ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey);
322

323
    double v1 = 0, v2 = 0, v = 0;
324
    if (prevRowIndex == -1) {
325
      SGroupKeys* p = taosArrayGet(pPrevValues, index);
326
      GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData);
327
    } else {
328
      GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex));
329 330
    }

331
    GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
332

333
#if 0
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352
    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            //            pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
353 354
#endif

X
Xiaoyu Wang 已提交
355 356 357
    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
    SPoint point = (SPoint){.key = windowKey, .val = &v};
358

X
Xiaoyu Wang 已提交
359
    taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
360

X
Xiaoyu Wang 已提交
361 362 363 364 365 366
    if (type == RESULT_ROW_START_INTERP) {
      pCtx[k].start.key = point.key;
      pCtx[k].start.val = v;
    } else {
      pCtx[k].end.key = point.key;
      pCtx[k].end.val = v;
367
    }
X
Xiaoyu Wang 已提交
368 369 370

    index += 1;
  }
371
#if 0
372
  }
373
#endif
374 375 376 377 378 379 380 381 382 383 384 385 386 387
}

static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

388 389
static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock,
                                              const TSKEY* tsCols, STimeWindow* win, SExprSupp* pSup) {
390
  bool ascQuery = (pInfo->inputOrder == TSDB_ORDER_ASC);
391

392
  TSKEY curTs = tsCols[pos];
393 394

  SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
X
Xiaoyu Wang 已提交
395
  TSKEY       lastTs = *(int64_t*)pTsKey->pData;
396 397 398 399 400

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery ? win->skey : win->ekey;
  if (key == curTs) {
401
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
402 403 404
    return true;
  }

405 406
  // it is the first time window, no need to do interpolation
  if (pTsKey->isNull && pos == 0) {
407
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
408 409
  } else {
    TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]);
410 411
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key,
                              RESULT_ROW_START_INTERP, pSup);
412 413 414 415 416
  }

  return true;
}

417 418 419
static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex,
                                            SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey,
                                            STimeWindow* win) {
420
  int32_t order = pInfo->inputOrder;
421 422

  TSKEY actualEndKey = tsCols[endRowIndex];
423
  TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey;
424 425

  // not ended in current data block, do not invoke interpolation
426
  if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) {
427
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
428 429 430
    return false;
  }

431
  // there is actual end point of current time window, no interpolation needs
432
  if (key == actualEndKey) {
433
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
434 435 436
    return true;
  }

437
  int32_t nextRowIndex = endRowIndex + 1;
438 439 440
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
441 442
  doTimeWindowInterpolation(pInfo->pPrevValues, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key,
                            RESULT_ROW_END_INTERP, pSup);
443 444 445
  return true;
}

446 447
bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd) {
  if (pInterval->interval != pInterval->sliding && (pWin->ekey < calStart || pWin->skey > calEnd)) {
5
54liuyao 已提交
448 449 450 451 452
    return false;
  }
  return true;
}

453 454 455 456
bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo) {
  return inCalSlidingWindow(pInterval, pWin, pBlockInfo->calWin.skey, pBlockInfo->calWin.ekey);
}

457
static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
5
54liuyao 已提交
458
                                      TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
X
Xiaoyu Wang 已提交
459
  bool ascQuery = (order == TSDB_ORDER_ASC);
460 461 462 463 464 465 466 467 468 469

  int32_t precision = pInterval->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
    return -1;
  }

5
54liuyao 已提交
470 471 472 473
  if (!inSlidingWindow(pInterval, pNext, pDataBlockInfo) && order == TSDB_ORDER_ASC) {
    return -1;
  }

474
  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
475 476 477 478
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
479
    startPos = prevPosition + 1;
480
  } else {
481
    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
482 483
      startPos = 0;
    } else {
484
      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
    }
  }

  /* interp query with fill should not skip time window */
  //  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
  //    return startPos;
  //  }

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
    if (ascQuery) {
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
      }
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
      }
    }
  }

  return startPos;
}

528
static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
529
  ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp == true;
  }
}

static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp = true;
  }
}

546 547
static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult,
                                        STimeWindow* win, int32_t startPos, int32_t forwardRows, SExprSupp* pSup) {
548
  if (!pInfo->timeWindowInterpo) {
549 550 551
    return;
  }

552
  ASSERT(pBlock != NULL);
553 554 555 556 557
  if (pBlock->pDataBlock == NULL) {
    //    tscError("pBlock->pDataBlock == NULL");
    return;
  }

558
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
559 560

  TSKEY* tsCols = (TSKEY*)(pColInfo->pData);
561
  bool   done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
562
  if (!done) {  // it is not interpolated, now start to generated the interpolated value
563
    bool interp = setTimeWindowInterpolationStartTs(pInfo, startPos, pBlock, tsCols, win, pSup);
564 565 566 567
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
568
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
569 570 571 572 573 574 575 576
  }

  // point interpolation does not require the end key time window interpolation.
  //  if (pointInterpQuery) {
  //    return;
  //  }

  // interpolation query does not generate the time window end interpolation
577
  done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
578
  if (!done) {
579
    int32_t endRowIndex = startPos + forwardRows - 1;
580

581
    TSKEY endKey = (pInfo->inputOrder == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
582
    bool  interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
583 584 585 586
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
587
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
588 589 590
  }
}

591 592
static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) {
  if (pBlock->pDataBlock == NULL) {
593 594 595
    return;
  }

596 597 598 599 600 601 602
  size_t num = taosArrayGetSize(pPrevKeys);
  for (int32_t k = 0; k < num; ++k) {
    SColumn* pc = taosArrayGet(pCols, k);

    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId);

    SGroupKeys* pkey = taosArrayGet(pPrevKeys, k);
X
Xiaoyu Wang 已提交
603
    for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) {
604 605 606 607 608 609 610
      if (colDataIsNull_s(pColInfo, i)) {
        continue;
      }

      char* val = colDataGetData(pColInfo, i);
      if (IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, varDataTLen(val));
611
        ASSERT(varDataTLen(val) <= pkey->bytes);
612 613 614 615 616 617
      } else {
        memcpy(pkey->pData, val, pkey->bytes);
      }

      break;
    }
618 619 620
  }
}

621 622 623 624
static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo,
                                       SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) {
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

625
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
626
  SExprSupp*                pSup = &pOperatorInfo->exprSupp;
627

L
Liu Jicong 已提交
628 629
  int32_t startPos = 0;
  int32_t numOfOutput = pSup->numOfExprs;
630

631
  SResultRow* pResult = NULL;
632

633
  while (1) {
L
Liu Jicong 已提交
634 635 636
    SListNode*          pn = tdListGetHead(pResultRowInfo->openWindow);
    SOpenWindowInfo*    pOpenWin = (SOpenWindowInfo*)pn->data;
    uint64_t            groupId = pOpenWin->groupId;
637
    SResultRowPosition* p1 = &pOpenWin->pos;
638 639 640
    if (p->pageId == p1->pageId && p->offset == p1->offset) {
      break;
    }
641

642
    SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1, false);
643 644 645
    if (NULL == pr) {
      T_LONG_JMP(pTaskInfo->env, terrno);
    }
646

647
    ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
648

649
    if (pr->closed) {
650
      ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
X
Xiaoyu Wang 已提交
651
             isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
652 653
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
654 655
      continue;
    }
656

657
    STimeWindow w = pr->win;
658 659
    int32_t     ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx,
                                             numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
660
    if (ret != TSDB_CODE_SUCCESS) {
S
Shengliang Guan 已提交
661
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
662 663
    }

664
    ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP));
665

X
Xiaoyu Wang 已提交
666 667
    SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
    int64_t     prevTs = *(int64_t*)pTsKey->pData;
H
Haojun Liao 已提交
668
    if (groupId == pBlock->info.id.groupId) {
669 670 671
      doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey,
                                RESULT_ROW_END_INTERP, pSup);
    }
672 673

    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
674
    setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
675

676
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, true);
L
Liu Jicong 已提交
677 678
    applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0,
                                    pBlock->info.rows, numOfExprs);
679 680 681

    if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
      closeResultRow(pr);
682 683
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
X
Xiaoyu Wang 已提交
684
    } else {  // the remains are can not be closed yet.
685
      break;
686
    }
687
  }
688
}
689

5
54liuyao 已提交
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745
typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);

int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;
      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;
      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;
      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

      if (comparefn(pKey, keyList, lastPos) > 0) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

5
54liuyao 已提交
746
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
747

X
Xiaoyu Wang 已提交
748 749 750
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;
5
54liuyao 已提交
751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (key >= getValuefn(keyList, lastPos)) return lastPos;
      if (key == getValuefn(keyList, firstPos)) return firstPos;
      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (key <= getValuefn(keyList, firstPos)) return firstPos;
      if (key == getValuefn(keyList, lastPos)) return lastPos;

      if (key > getValuefn(keyList, lastPos)) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
797 798
  }

5
54liuyao 已提交
799 800 801
  return midPos;
}

5
54liuyao 已提交
802
int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {
L
Liu Jicong 已提交
803
  SArray*          res = (SArray*)data;
5
54liuyao 已提交
804
  SPullWindowInfo* pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
805
  SPullWindowInfo* pData = (SPullWindowInfo*)pKey;
5
54liuyao 已提交
806
  if (pData->groupId > pos->groupId) {
5
54liuyao 已提交
807
    return 1;
5
54liuyao 已提交
808 809 810 811 812 813 814 815
  } else if (pData->groupId < pos->groupId) {
    return -1;
  }

  if (pData->window.skey > pos->window.ekey) {
    return 1;
  } else if (pData->window.ekey < pos->window.skey) {
    return -1;
5
54liuyao 已提交
816
  }
5
54liuyao 已提交
817
  return 0;
5
54liuyao 已提交
818 819 820 821 822 823 824 825
}

static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {
  int32_t size = taosArrayGetSize(pPullWins);
  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);
  if (index == -1) {
    index = 0;
  } else {
5
54liuyao 已提交
826 827
    int32_t code = comparePullWinKey(pPullInfo, pPullWins, index);
    if (code == 0) {
L
Liu Jicong 已提交
828
      SPullWindowInfo* pos = taosArrayGet(pPullWins, index);
5
54liuyao 已提交
829 830 831 832
      pos->window.skey = TMIN(pos->window.skey, pPullInfo->window.skey);
      pos->window.ekey = TMAX(pos->window.ekey, pPullInfo->window.ekey);
      pos->calWin.skey = TMIN(pos->calWin.skey, pPullInfo->calWin.skey);
      pos->calWin.ekey = TMAX(pos->calWin.ekey, pPullInfo->calWin.ekey);
5
54liuyao 已提交
833
      return TSDB_CODE_SUCCESS;
L
Liu Jicong 已提交
834
    } else if (code > 0) {
5
54liuyao 已提交
835
      index++;
5
54liuyao 已提交
836 837 838 839 840 841 842 843
    }
  }
  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
844 845 846
static int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) {
  winInfo.sessionWin.win.ekey = winInfo.sessionWin.win.skey;
  return tSimpleHashPut(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));
5
54liuyao 已提交
847 848
}

5
54liuyao 已提交
849 850
static int32_t saveWinResult(SWinKey* pKey, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) {
  tSimpleHashPut(pUpdatedMap, pKey, sizeof(SWinKey), &pPos, POINTER_BYTES);
5
54liuyao 已提交
851
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
852 853
}

5
54liuyao 已提交
854 855 856 857
static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) {
  SWinKey key = {.ts = ts, .groupId = groupId};
  saveWinResult(&key, pPos, pUpdatedMap);
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
858 859
}

860
static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) {
5
54liuyao 已提交
861 862
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
H
Haojun Liao 已提交
863
    SWinKey* pW = taosArrayGet(pWins, i);
864
    void*    tmp = tSimpleHashGet(pUpdatedMap, pW, sizeof(SWinKey));
5
54liuyao 已提交
865 866 867
    if (tmp) {
      void* value = *(void**)tmp;
      taosMemoryFree(value);
868
      tSimpleHashRemove(pUpdatedMap, pW, sizeof(SWinKey));
5
54liuyao 已提交
869
    }
5
54liuyao 已提交
870 871 872
  }
}

873
int32_t compareWinKey(void* pKey, void* data, int32_t index) {
L
liuyao 已提交
874 875
  void* pDataPos = taosArrayGet((SArray*)data, index);
  return winKeyCmprImpl(pKey, pDataPos);
5
54liuyao 已提交
876 877
}

878
static void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins) {
5
54liuyao 已提交
879 880
  taosArraySort(pDelWins, winKeyCmprImpl);
  taosArrayRemoveDuplicate(pDelWins, winKeyCmprImpl, NULL);
L
Liu Jicong 已提交
881
  int32_t delSize = taosArrayGetSize(pDelWins);
882
  if (tSimpleHashGetSize(pUpdatedMap) == 0 || delSize == 0) {
5
54liuyao 已提交
883
    return;
dengyihao's avatar
dengyihao 已提交
884
  }
885 886 887 888 889 890
  void*   pIte = NULL;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pUpdatedMap, pIte, &iter)) != NULL) {
    SWinKey* pResKey = tSimpleHashGetKey(pIte, NULL);
    int32_t  index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinKey);
    if (index >= 0 && 0 == compareWinKey(pResKey, pDelWins, index)) {
891
      taosArrayRemove(pDelWins, index);
892
      delSize = taosArrayGetSize(pDelWins);
893 894 895 896
    }
  }
}

5
54liuyao 已提交
897
bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) {
5
54liuyao 已提交
898
  ASSERTS(pTwSup->maxTs == INT64_MIN || pTwSup->maxTs > 0, "maxts should greater than 0");
5
54liuyao 已提交
899
  return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark;
5
54liuyao 已提交
900 901
}

5
54liuyao 已提交
902 903 904 905 906
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); }

bool needDeleteWindowBuf(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) {
  return pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark;
}
5
54liuyao 已提交
907

5
54liuyao 已提交
908
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
909
                            int32_t scanFlag) {
910
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
911

912
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
913
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
914

X
Xiaoyu Wang 已提交
915
  int32_t     startPos = 0;
916
  int32_t     numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
917
  int64_t*    tsCols = extractTsCol(pBlock, pInfo);
H
Haojun Liao 已提交
918
  uint64_t    tableGroupId = pBlock->info.id.groupId;
919
  bool        ascScan = (pInfo->inputOrder == TSDB_ORDER_ASC);
X
Xiaoyu Wang 已提交
920 921
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;
922

923 924
  STimeWindow win =
      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder);
925 926
  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
927
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
S
Shengliang Guan 已提交
928
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
929
  }
930

X
Xiaoyu Wang 已提交
931 932
  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
933
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
934 935

  // prev time window not interpolation yet.
936
  if (pInfo->timeWindowInterpo) {
937
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
938
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);
939 940

    // restore current time window
941 942
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
943
    if (ret != TSDB_CODE_SUCCESS) {
S
Shengliang Guan 已提交
944
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
945 946
    }

947
    // window start key interpolation
948
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
949
  }
950

951
  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
L
Liu Jicong 已提交
952 953
  applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                                  pBlock->info.rows, numOfOutput);
954 955

  doCloseWindow(pResultRowInfo, pInfo, pResult);
956 957 958

  STimeWindow nextWin = win;
  while (1) {
959
    int32_t prevEndPos = forwardRows - 1 + startPos;
960
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->inputOrder);
961 962 963 964
    if (startPos < 0) {
      break;
    }
    // null data, failed to allocate more memory buffer
X
Xiaoyu Wang 已提交
965
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
966
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
967
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
S
Shengliang Guan 已提交
968
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
969 970
    }

X
Xiaoyu Wang 已提交
971
    ekey = ascScan ? nextWin.ekey : nextWin.skey;
972
    forwardRows =
973
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
974
    // window start(end) key interpolation
975
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
L
Liu Jicong 已提交
976
    // TODO: add to open window? how to close the open windows after input blocks exhausted?
S
shenglian zhou 已提交
977
#if 0
978 979 980 981
    if ((ascScan && ekey <= pBlock->info.window.ekey) ||
        (!ascScan && ekey >= pBlock->info.window.skey)) {
      // window start(end) key interpolation
      doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
982
    } else if (pInfo->timeWindowInterpo) {
983 984
      addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
    }
S
shenglian zhou 已提交
985
#endif
986
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
L
Liu Jicong 已提交
987 988
    applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                                    pBlock->info.rows, numOfOutput);
989
    doCloseWindow(pResultRowInfo, pInfo, pResult);
990 991 992
  }

  if (pInfo->timeWindowInterpo) {
993
    saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
994
  }
995 996 997 998 999 1000
}

void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
  // current result is done in computing final results.
  if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
    closeResultRow(pResult);
1001
    SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
D
dapan1121 已提交
1002
    taosMemoryFree(pNode);
1003 1004 1005
  }
}

1006 1007 1008 1009 1010
SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, uint64_t groupId) {
  SOpenWindowInfo openWin = {0};
  openWin.pos.pageId = pResult->pageId;
  openWin.pos.offset = pResult->offset;
  openWin.groupId = groupId;
L
Liu Jicong 已提交
1011
  SListNode* pn = tdListGetTail(pResultRowInfo->openWindow);
1012
  if (pn == NULL) {
1013 1014
    tdListAppend(pResultRowInfo->openWindow, &openWin);
    return openWin.pos;
1015 1016
  }

L
Liu Jicong 已提交
1017
  SOpenWindowInfo* px = (SOpenWindowInfo*)pn->data;
1018 1019
  if (px->pos.pageId != openWin.pos.pageId || px->pos.offset != openWin.pos.offset || px->groupId != openWin.groupId) {
    tdListAppend(pResultRowInfo->openWindow, &openWin);
1020 1021
  }

1022
  return openWin.pos;
1023 1024 1025 1026
}

int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
  TSKEY* tsCols = NULL;
1027

D
dapan1121 已提交
1028
  if (pBlock->pDataBlock != NULL && pBlock->info.dataLoad) {
1029 1030
    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
H
Haojun Liao 已提交
1031
    ASSERT(tsCols[0] != 0);
1032

1033 1034 1035 1036 1037 1038
    // no data in primary ts
    if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) {
      return NULL;
    }

    if (tsCols[0] != 0 && (pBlock->info.window.skey == 0 && pBlock->info.window.ekey == 0)) {
1039 1040 1041 1042 1043
      blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
    }
  }

  return tsCols;
1044 1045 1046 1047 1048 1049 1050
}

static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
  }

1051 1052 1053
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  SOperatorInfo* downstream = pOperator->pDownstream[0];

1054
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
1055
  SExprSupp*                pSup = &pOperator->exprSupp;
1056

1057
  int32_t scanFlag = MAIN_SCAN;
1058
  int64_t st = taosGetTimestampUs();
1059 1060

  while (1) {
1061
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1062 1063 1064 1065
    if (pBlock == NULL) {
      break;
    }

1066
    getTableScanInfo(pOperator, &pInfo->inputOrder, &scanFlag, true);
1067

1068
    if (pInfo->scalarSupp.pExprInfo != NULL) {
L
Liu Jicong 已提交
1069 1070
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
1071 1072
    }

1073
    // the pDataBlock are always the same one, no need to call this again
1074
    setInputDataBlock(pSup, pBlock, pInfo->inputOrder, scanFlag, true);
1075
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag);
1076 1077
  }

1078
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->resultTsOrder);
1079
  OPTR_SET_OPENED(pOperator);
1080 1081

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1082 1083 1084
  return TSDB_CODE_SUCCESS;
}

1085 1086 1087 1088 1089
static bool compareVal(const char* v, const SStateKeys* pKey) {
  if (IS_VAR_DATA_TYPE(pKey->type)) {
    if (varDataLen(v) != varDataLen(pKey->pData)) {
      return false;
    } else {
D
dapan1121 已提交
1090
      return memcmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0;
1091 1092 1093 1094 1095 1096
    }
  } else {
    return memcmp(pKey->pData, v, pKey->bytes) == 0;
  }
}

1097
static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
L
Liu Jicong 已提交
1098
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1099
  SExprSupp*     pSup = &pOperator->exprSupp;
1100

1101
  SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
H
Haojun Liao 已提交
1102
  int64_t          gid = pBlock->info.id.groupId;
1103 1104

  bool    masterScan = true;
1105
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1106 1107
  int16_t bytes = pStateColInfoData->info.bytes;

1108
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1109 1110 1111 1112 1113
  TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

1114
  struct SColumnDataAgg* pAgg = NULL;
1115
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
X
Xiaoyu Wang 已提交
1116
    pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
1117
    if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
1118 1119 1120 1121 1122
      continue;
    }

    char* val = colDataGetData(pStateColInfoData, j);

1123
    if (gid != pRowSup->groupId || !pInfo->hasKey) {
1124 1125 1126 1127 1128 1129 1130
      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }

1131 1132
      pInfo->hasKey = true;

1133 1134
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1135
    } else if (compareVal(val, &pInfo->stateKey)) {
1136
      doKeepTuple(pRowSup, tsList[j], gid);
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // a new state window started
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1147 1148
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1149
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
S
Shengliang Guan 已提交
1150
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR);
1151 1152 1153
      }

      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
H
Haojun Liao 已提交
1154
      applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
L
Liu Jicong 已提交
1155
                                      pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1156 1157

      // here we start a new session window
1158 1159
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1160 1161 1162 1163 1164 1165 1166

      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }
1167 1168 1169 1170 1171
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1172 1173
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1174
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
S
Shengliang Guan 已提交
1175
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR);
1176 1177 1178
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
L
Liu Jicong 已提交
1179 1180
  applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                                  pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1181 1182
}

H
Hongze Cheng 已提交
1183
static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) {
1184 1185
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
1186 1187 1188
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
1189
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1190

1191 1192 1193
  SExprSupp* pSup = &pOperator->exprSupp;
  int32_t    order = TSDB_ORDER_ASC;
  int64_t    st = taosGetTimestampUs();
1194 1195 1196

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  while (1) {
1197
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1198 1199 1200 1201
    if (pBlock == NULL) {
      break;
    }

1202
    setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
1203 1204
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1205 1206 1207 1208 1209 1210 1211 1212 1213
    // there is an scalar expression that needs to be calculated right before apply the group aggregation.
    if (pInfo->scalarSup.pExprInfo != NULL) {
      pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
                                              pInfo->scalarSup.numOfExprs, NULL);
      if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
        T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
      }
    }

1214 1215 1216
    doStateWindowAggImpl(pOperator, pInfo, pBlock);
  }

X
Xiaoyu Wang 已提交
1217
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1218
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1219 1220
  pOperator->status = OP_RES_TO_RETURN;

1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
  return TSDB_CODE_SUCCESS;
}

static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
  SOptrBasicInfo*           pBInfo = &pInfo->binfo;

  pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
  if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
H
Haojun Liao 已提交
1235
    setOperatorCompleted(pOperator);
1236 1237 1238
    return NULL;
  }

1239
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1240
  while (1) {
1241
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
H
Haojun Liao 已提交
1242
    doFilter(pBInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
1243

1244
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1245
    if (!hasRemain) {
H
Haojun Liao 已提交
1246
      setOperatorCompleted(pOperator);
1247 1248
      break;
    }
1249

1250 1251 1252 1253
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
1254

1255
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1256
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1257 1258
}

1259
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
1260
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
1261
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1262 1263 1264 1265 1266 1267

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pBlock = pInfo->binfo.pRes;
1268 1269 1270 1271
  pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
  if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
    return NULL;
  }
1272

1273 1274 1275
  while (1) {
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
    doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL);
1276

1277 1278
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
    if (!hasRemain) {
H
Haojun Liao 已提交
1279
      setOperatorCompleted(pOperator);
1280
      break;
1281 1282
    }

1283 1284 1285
    if (pBlock->info.rows > 0) {
      break;
    }
1286
  }
1287 1288 1289 1290 1291

  size_t rows = pBlock->info.rows;
  pOperator->resultInfo.totalRows += rows;

  return (rows == 0) ? NULL : pBlock;
1292 1293
}

5
54liuyao 已提交
1294
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
L
Liu Jicong 已提交
1295
  for (int i = 0; i < num; i++) {
5
54liuyao 已提交
1296 1297
    if (type == STREAM_INVERT) {
      fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
L
Liu Jicong 已提交
1298
    } else if (type == STREAM_NORMAL) {
5
54liuyao 已提交
1299 1300 1301 1302
      fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
    }
  }
}
5
54liuyao 已提交
1303

5
54liuyao 已提交
1304
static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) {
1305
  SResultRow* pResult = getResultRowByPos(pResultBuf, p1, false);
1306 1307 1308
  if (NULL == pResult) {
    return;
  }
1309

1310
  SqlFunctionCtx* pCtx = pSup->pCtx;
5
54liuyao 已提交
1311
  for (int32_t i = 0; i < numOfOutput; ++i) {
1312
    pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
1313 1314 1315 1316 1317 1318 1319 1320 1321
    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
      continue;
    }
    pResInfo->initialized = false;
    if (pCtx[i].functionId != -1) {
      pCtx[i].fpSet.init(&pCtx[i], pResInfo);
    }
  }
5
54liuyao 已提交
1322
  SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
1323 1324 1325
  if (NULL == bufPage) {
    return;
  }
5
54liuyao 已提交
1326 1327
  setBufPageDirty(bufPage, true);
  releaseBufPage(pResultBuf, bufPage);
5
54liuyao 已提交
1328 1329
}

1330
static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) {
5
54liuyao 已提交
1331 1332 1333
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
  SWinKey                      key = {.ts = ts, .groupId = groupId};
  tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));
1334
  streamStateDel(pInfo->pState, &key);
5
54liuyao 已提交
1335 1336 1337
  return true;
}

1338
static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins,
1339
                            SSHashObj* pUpdatedMap) {
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
  SColumnInfoData*             pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  TSKEY*                       startTsCols = (TSKEY*)pStartTsCol->pData;
  SColumnInfoData*             pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*                       endTsCols = (TSKEY*)pEndTsCol->pData;
  SColumnInfoData*             pCalStTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
  TSKEY*                       calStTsCols = (TSKEY*)pCalStTsCol->pData;
  SColumnInfoData*             pCalEnTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
  TSKEY*                       calEnTsCols = (TSKEY*)pCalEnTsCol->pData;
  SColumnInfoData*             pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*                    pGpDatas = (uint64_t*)pGpCol->pData;
5
54liuyao 已提交
1351
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
H
Haojun Liao 已提交
1352
    SResultRowInfo dumyInfo = {0};
5
54liuyao 已提交
1353
    dumyInfo.cur.pageId = -1;
H
Haojun Liao 已提交
1354

1355 1356 1357 1358 1359 1360 1361 1362
    STimeWindow win = {0};
    if (IS_FINAL_OP(pInfo)) {
      win.skey = startTsCols[i];
      win.ekey = endTsCols[i];
    } else {
      win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
    }

5
54liuyao 已提交
1363
    do {
1364 1365 1366 1367
      if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i])) {
        getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
        continue;
      }
5
54liuyao 已提交
1368 1369
      uint64_t winGpId = pGpDatas[i];
      SWinKey  winRes = {.ts = win.skey, .groupId = winGpId};
dengyihao's avatar
dengyihao 已提交
1370
      void*    chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
L
liuyao 已提交
1371 1372 1373 1374
      if (chIds) {
        getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
        continue;
      }
dengyihao's avatar
dengyihao 已提交
1375
      bool res = doDeleteWindow(pOperator, win.skey, winGpId);
5
54liuyao 已提交
1376 1377 1378 1379
      if (pUpWins && res) {
        taosArrayPush(pUpWins, &winRes);
      }
      if (pUpdatedMap) {
1380
        tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
1381 1382
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
5
54liuyao 已提交
1383
    } while (win.ekey <= endTsCols[i]);
5
54liuyao 已提交
1384 1385 1386
  }
}

1387
static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SSHashObj* resWins) {
1388 1389 1390
  void*   pIte = NULL;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
1391 1392 1393
    SWinKey* pKey = tSimpleHashGetKey(pIte, NULL);
    uint64_t groupId = pKey->groupId;
    TSKEY    ts = pKey->ts;
5
54liuyao 已提交
1394
    int32_t  code = saveWinResultInfo(ts, groupId, *(SRowBuffPos**)pIte, resWins);
5
54liuyao 已提交
1395 1396 1397 1398 1399 1400 1401
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
1402
static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval,
1403
                                         SHashObj* pPullDataMap, SSHashObj* closeWins, SArray* pDelWins,
1404
                                         SOperatorInfo* pOperator) {
5
54liuyao 已提交
1405
  qDebug("===stream===close interval window");
1406 1407 1408
  void*                        pIte = NULL;
  int32_t                      iter = 0;
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
1409
  int32_t                      delSize = taosArrayGetSize(pDelWins);
5
54liuyao 已提交
1410
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
1411
    void*    key = tSimpleHashGetKey(pIte, NULL);
1412 1413 1414 1415 1416 1417 1418 1419 1420
    SWinKey* pWinKey = (SWinKey*)key;
    if (delSize > 0) {
      int32_t index = binarySearchCom(pDelWins, delSize, pWinKey, TSDB_ORDER_DESC, compareWinKey);
      if (index >= 0 && 0 == compareWinKey(pWinKey, pDelWins, index)) {
        taosArrayRemove(pDelWins, index);
        delSize = taosArrayGetSize(pDelWins);
      }
    }

5
54liuyao 已提交
1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439
    void*       chIds = taosHashGet(pPullDataMap, pWinKey, sizeof(SWinKey));
    STimeWindow win = {
        .skey = pWinKey->ts,
        .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1,
    };
    if (isCloseWindow(&win, pTwSup)) {
      if (chIds && pPullDataMap) {
        SArray* chAy = *(SArray**)chIds;
        int32_t size = taosArrayGetSize(chAy);
        qDebug("===stream===window %" PRId64 " wait child size:%d", pWinKey->ts, size);
        for (int32_t i = 0; i < size; i++) {
          qDebug("===stream===window %" PRId64 " wait child id:%d", pWinKey->ts, *(int32_t*)taosArrayGet(chAy, i));
        }
        continue;
      } else if (pPullDataMap) {
        qDebug("===stream===close window %" PRId64, pWinKey->ts);
      }

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
1440
        int32_t code = saveWinResult(pWinKey, *(SRowBuffPos**)pIte, closeWins);
5
54liuyao 已提交
1441 1442 1443 1444 1445
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
      }
      tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter);
1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
    }
  }
  return TSDB_CODE_SUCCESS;
}

STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
  return w;
}

L
liuyao 已提交
1457 1458
static void deleteIntervalDiscBuf(SStreamState* pState, TSKEY mark) {
  //todo(liuyao) delete expired check point
5
54liuyao 已提交
1459 1460
}

1461
static void closeChildIntervalWindow(SOperatorInfo* pOperator, SArray* pChildren, TSKEY maxTs) {
5
54liuyao 已提交
1462 1463
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
1464 1465
    SOperatorInfo*               pChildOp = taosArrayGetP(pChildren, i);
    SStreamIntervalOperatorInfo* pChInfo = pChildOp->info;
5
54liuyao 已提交
1466
    ASSERTS(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE, "children trigger type should be at once");
5
54liuyao 已提交
1467
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
1468
    closeStreamIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL,
1469
                              NULL, pOperator);
1470 1471 1472
  }
}

1473 1474
static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWins, int32_t* index,
                                SSDataBlock* pBlock) {
1475 1476 1477 1478 1479 1480 1481 1482
  blockDataCleanup(pBlock);
  int32_t size = taosArrayGetSize(pWins);
  if (*index == size) {
    *index = 0;
    taosArrayClear(pWins);
    return;
  }
  blockDataEnsureCapacity(pBlock, size - *index);
1483
  uint64_t uid = 0;
1484
  for (int32_t i = *index; i < size; i++) {
H
Haojun Liao 已提交
1485
    SWinKey* pWin = taosArrayGet(pWins, i);
1486 1487
    void*    tbname = NULL;
    streamStateGetParName(pInfo->pState, pWin->groupId, &tbname);
1488 1489 1490 1491 1492 1493 1494
    if (tbname == NULL) {
      appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL);
    } else {
      char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
      STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
      appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName);
    }
dengyihao's avatar
dengyihao 已提交
1495
    streamFreeVal(tbname);
1496
    (*index)++;
5
54liuyao 已提交
1497 1498 1499
  }
}

1500
static void destroyStateWindowOperatorInfo(void* param) {
1501
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
1502
  cleanupBasicInfo(&pInfo->binfo);
1503
  taosMemoryFreeClear(pInfo->stateKey.pData);
1504
  cleanupExprSupp(&pInfo->scalarSup);
D
dapan1121 已提交
1505 1506 1507
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  cleanupAggSup(&pInfo->aggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
1508

D
dapan1121 已提交
1509
  taosMemoryFreeClear(param);
1510 1511
}

H
Haojun Liao 已提交
1512
static void freeItem(void* param) {
L
Liu Jicong 已提交
1513
  SGroupKeys* pKey = (SGroupKeys*)param;
H
Haojun Liao 已提交
1514 1515 1516
  taosMemoryFree(pKey->pData);
}

1517
void destroyIntervalOperatorInfo(void* param) {
1518
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
1519
  cleanupBasicInfo(&pInfo->binfo);
1520
  cleanupAggSup(&pInfo->aggSup);
1521 1522 1523 1524
  cleanupExprSupp(&pInfo->scalarSupp);

  tdListFree(pInfo->binfo.resultRowInfo.openWindow);

H
Haojun Liao 已提交
1525 1526 1527 1528
  pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols);
  taosArrayDestroyEx(pInfo->pPrevValues, freeItem);

  pInfo->pPrevValues = NULL;
1529

H
Haojun Liao 已提交
1530 1531
  cleanupGroupResInfo(&pInfo->groupResInfo);
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
D
dapan1121 已提交
1532
  taosMemoryFreeClear(param);
1533 1534
}

1535
void destroyStreamFinalIntervalOperatorInfo(void* param) {
1536
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;
1537
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
1538
  cleanupAggSup(&pInfo->aggSup);
L
Liu Jicong 已提交
1539
  // it should be empty.
5
54liuyao 已提交
1540 1541 1542
  taosHashCleanup(pInfo->pPullDataMap);
  taosArrayDestroy(pInfo->pPullWins);
  blockDataDestroy(pInfo->pPullDataRes);
L
Liu Jicong 已提交
1543 1544
  taosArrayDestroy(pInfo->pDelWins);
  blockDataDestroy(pInfo->pDelRes);
5
54liuyao 已提交
1545
  streamFileStateDestroy(pInfo->pState->pFileState);
1546
  taosMemoryFreeClear(pInfo->pState);
5
54liuyao 已提交
1547

1548 1549 1550 1551
  if (pInfo->pChildren) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
5
54liuyao 已提交
1552
      destroyOperatorInfo(pChildOp);
1553
    }
L
Liu Jicong 已提交
1554
    taosArrayDestroy(pInfo->pChildren);
1555
  }
1556
  nodesDestroyNode((SNode*)pInfo->pPhyNode);
5
54liuyao 已提交
1557
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
1558
  pInfo->groupResInfo.pRows = taosArrayDestroy(pInfo->groupResInfo.pRows);
5
54liuyao 已提交
1559
  cleanupExprSupp(&pInfo->scalarSupp);
1560

D
dapan1121 已提交
1561
  taosMemoryFreeClear(param);
5
54liuyao 已提交
1562 1563
}

1564
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
5
54liuyao 已提交
1565
  for (int32_t i = 0; i < numOfCols; i++) {
5
54liuyao 已提交
1566
    if (fmIsUserDefinedFunc(pFCtx[i].functionId) || !fmIsInvertible(pFCtx[i].functionId)) {
5
54liuyao 已提交
1567 1568 1569 1570 1571 1572
      return false;
    }
  }
  return true;
}

1573
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
1574 1575 1576
  // the primary timestamp column
  bool needed = false;

L
Liu Jicong 已提交
1577
  for (int32_t i = 0; i < numOfCols; ++i) {
1578
    SExprInfo* pExpr = pCtx[i].pExpr;
H
Haojun Liao 已提交
1579
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1580
      needed = true;
H
Haojun Liao 已提交
1581
      break;
1582 1583 1584
    }
  }

H
Haojun Liao 已提交
1585 1586 1587
  if (needed) {
    pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
    pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
1588

H
Haojun Liao 已提交
1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
    {  // ts column
      SColumn c = {0};
      c.colId = 1;
      c.slotId = pInfo->primaryTsIndex;
      c.type = TSDB_DATA_TYPE_TIMESTAMP;
      c.bytes = sizeof(int64_t);
      taosArrayPush(pInfo->pInterpCols, &c);

      SGroupKeys key;
      key.bytes = c.bytes;
      key.type = c.type;
      key.isNull = true;  // to denote no value is assigned yet
      key.pData = taosMemoryCalloc(1, c.bytes);
      taosArrayPush(pInfo->pPrevValues, &key);
    }
1604 1605
  }

X
Xiaoyu Wang 已提交
1606
  for (int32_t i = 0; i < numOfCols; ++i) {
1607 1608
    SExprInfo* pExpr = pCtx[i].pExpr;

H
Haojun Liao 已提交
1609
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1610 1611 1612
      SFunctParam* pParam = &pExpr->base.pParam[0];

      SColumn c = *pParam->pCol;
1613
      taosArrayPush(pInfo->pInterpCols, &c);
1614 1615

      SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1616 1617
      key.bytes = c.bytes;
      key.type = c.type;
1618
      key.isNull = false;
X
Xiaoyu Wang 已提交
1619
      key.pData = taosMemoryCalloc(1, c.bytes);
1620
      taosArrayPush(pInfo->pPrevValues, &key);
1621 1622 1623 1624 1625 1626
    }
  }

  return needed;
}

L
Liu Jicong 已提交
1627
void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval,
5
54liuyao 已提交
1628
                            STimeWindowAggSupp* pTwSup) {
1629
  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
5
54liuyao 已提交
1630
    initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, pTwSup);
1631 1632
    return;
  }
5
54liuyao 已提交
1633
  SStreamScanInfo* pScanInfo = downstream->info;
1634 1635
  pScanInfo->windowSup.parentType = type;
  pScanInfo->windowSup.pIntervalAggSup = pSup;
5
54liuyao 已提交
1636
  if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) {
5
54liuyao 已提交
1637 1638
    pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, pTwSup->waterMark);
  }
1639
  pScanInfo->interval = *pInterval;
5
54liuyao 已提交
1640
  pScanInfo->twAggSup = *pTwSup;
5
54liuyao 已提交
1641 1642
}

H
Haojun Liao 已提交
1643 1644
void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
  for (int32_t i = 0; i < numOfExpr; i++) {
L
Liu Jicong 已提交
1645
    //    pCtx[i].isStream = true;
H
Haojun Liao 已提交
1646 1647 1648
  }
}

H
Haojun Liao 已提交
1649
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode,
5
54liuyao 已提交
1650
                                          SExecTaskInfo* pTaskInfo) {
1651
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1652
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1653 1654 1655 1656
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

H
Haojun Liao 已提交
1657
  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->window.node.pOutputDataBlockDesc);
H
Haojun Liao 已提交
1658 1659 1660 1661 1662 1663
  initBasicInfo(&pInfo->binfo, pResBlock);

  SExprSupp* pSup = &pOperator->exprSupp;
  pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId;

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
H
Haojun Liao 已提交
1664 1665
  initResultSizeInfo(&pOperator->resultInfo, 512);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
H
Haojun Liao 已提交
1666 1667 1668

  int32_t    num = 0;
  SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
L
Liu Jicong 已提交
1669 1670
  int32_t    code =
      initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState);
H
Haojun Liao 已提交
1671 1672 1673 1674 1675
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  SInterval interval = {.interval = pPhyNode->interval,
1676 1677 1678 1679 1680
                        .sliding = pPhyNode->sliding,
                        .intervalUnit = pPhyNode->intervalUnit,
                        .slidingUnit = pPhyNode->slidingUnit,
                        .offset = pPhyNode->offset,
                        .precision = ((SColumnNode*)pPhyNode->window.pTspk)->node.resType.precision};
H
Haojun Liao 已提交
1681 1682 1683 1684 1685 1686 1687

  STimeWindowAggSupp as = {
      .waterMark = pPhyNode->window.watermark,
      .calTrigger = pPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
  };

L
Liu Jicong 已提交
1688
  pInfo->win = pTaskInfo->window;
1689 1690
  pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
  pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
H
Haojun Liao 已提交
1691 1692
  pInfo->interval = interval;
  pInfo->twAggSup = as;
1693
  pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock;
1694 1695 1696 1697

  if (pPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
H
Haojun Liao 已提交
1698
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
1699 1700 1701 1702
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
1703

H
Haojun Liao 已提交
1704 1705 1706 1707 1708
  code = filterInitFromNode((SNode*)pPhyNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

1709
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
H
Haojun Liao 已提交
1710
  pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, pInfo);
1711
  if (pInfo->timeWindowInterpo) {
1712
    pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
H
Haojun Liao 已提交
1713 1714 1715
    if (pInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
1716
  }
1717

1718
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
L
Liu Jicong 已提交
1719 1720
  setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED,
                  pInfo, pTaskInfo);
1721

L
Liu Jicong 已提交
1722 1723
  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo,
                                         optrDefaultBufFn, NULL);
1724 1725 1726 1727 1728 1729 1730 1731

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1732
_error:
H
Haojun Liao 已提交
1733 1734 1735
  if (pInfo != NULL) {
    destroyIntervalOperatorInfo(pInfo);
  }
1736 1737 1738 1739 1740
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

1741
// todo handle multiple timeline cases. assume no timeline interweaving
1742 1743
static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1744
  SExprSupp*     pSup = &pOperator->exprSupp;
1745

1746
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1747 1748

  bool    masterScan = true;
1749
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
H
Haojun Liao 已提交
1750
  int64_t gid = pBlock->info.id.groupId;
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764

  int64_t gap = pInfo->gap;

  if (!pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->winSup.prevTs = INT64_MIN;
  }

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

  // In case of ascending or descending order scan data, only one time window needs to be kepted for each table.
  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
1765 1766 1767
    if (gid != pRowSup->groupId || pInfo->winSup.prevTs == INT64_MIN) {
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
H
Haojun Liao 已提交
1768 1769
    } else if (((tsList[j] - pRowSup->prevTs >= 0) && (tsList[j] - pRowSup->prevTs <= gap)) ||
               ((pRowSup->prevTs - tsList[j] >= 0) && (pRowSup->prevTs - tsList[j] <= gap))) {
1770
      // The gap is less than the threshold, so it belongs to current session window that has been opened already.
1771
      doKeepTuple(pRowSup, tsList[j], gid);
1772 1773 1774 1775 1776 1777 1778 1779 1780 1781
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1782 1783
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1784
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
S
Shengliang Guan 已提交
1785
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR);
1786 1787 1788 1789
      }

      // pInfo->numOfRows data belong to the current session window
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
H
Haojun Liao 已提交
1790
      applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
L
Liu Jicong 已提交
1791
                                      pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1792 1793

      // here we start a new session window
1794 1795
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1796 1797 1798 1799 1800
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1801 1802
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1803
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
S
Shengliang Guan 已提交
1804
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR);
1805 1806 1807
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
L
Liu Jicong 已提交
1808 1809
  applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                                  pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1810 1811
}

1812
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
1813 1814 1815 1816 1817 1818
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*          pBInfo = &pInfo->binfo;
1819
  SExprSupp*               pSup = &pOperator->exprSupp;
1820 1821

  if (pOperator->status == OP_RES_TO_RETURN) {
1822
    while (1) {
1823
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
H
Haojun Liao 已提交
1824
      doFilter(pBInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
1825

1826
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1827
      if (!hasRemain) {
H
Haojun Liao 已提交
1828
        setOperatorCompleted(pOperator);
1829 1830
        break;
      }
1831

1832 1833 1834 1835 1836
      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1837
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1838 1839
  }

1840 1841 1842
  int64_t st = taosGetTimestampUs();
  int32_t order = TSDB_ORDER_ASC;

1843 1844 1845
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1846
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1847 1848 1849 1850 1851
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1852
    setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
1853 1854
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1855 1856 1857
    doSessionWindowAggImpl(pOperator, pInfo, pBlock);
  }

1858 1859
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;

1860 1861 1862
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;

1863
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1864
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1865
  while (1) {
1866
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
H
Haojun Liao 已提交
1867
    doFilter(pBInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
1868

1869
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1870
    if (!hasRemain) {
H
Haojun Liao 已提交
1871
      setOperatorCompleted(pOperator);
1872 1873
      break;
    }
1874

1875 1876 1877 1878 1879
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1880
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1881 1882
}

H
Haojun Liao 已提交
1883
// todo make this as an non-blocking operator
1884 1885
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
                                             SExecTaskInfo* pTaskInfo) {
1886 1887 1888 1889 1890 1891
  SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1892 1893 1894
  int32_t      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;

1895 1896 1897
  if (pStateNode->window.pExprs != NULL) {
    int32_t    numOfScalarExpr = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalarExpr);
H
Hongze Cheng 已提交
1898
    int32_t    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
1899 1900 1901 1902 1903
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

1904
  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
1905 1906 1907 1908 1909 1910 1911
  pInfo->stateKey.type = pInfo->stateCol.type;
  pInfo->stateKey.bytes = pInfo->stateCol.bytes;
  pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
  if (pInfo->stateKey.pData == NULL) {
    goto _error;
  }

H
Haojun Liao 已提交
1912 1913 1914 1915 1916
  int32_t code = filterInitFromNode((SNode*)pStateNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

1917 1918
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

1919 1920
  int32_t    num = 0;
  SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num);
1921
  initResultSizeInfo(&pOperator->resultInfo, 4096);
H
Haojun Liao 已提交
1922

L
Liu Jicong 已提交
1923 1924
  code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
                    pTaskInfo->streamInfo.pState);
1925 1926 1927 1928
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
1929
  SSDataBlock* pResBlock = createDataBlockFromDescNode(pStateNode->window.node.pOutputDataBlockDesc);
1930
  initBasicInfo(&pInfo->binfo, pResBlock);
1931
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1932

L
Liu Jicong 已提交
1933 1934
  pInfo->twAggSup =
      (STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType};
1935

1936 1937
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

X
Xiaoyu Wang 已提交
1938
  pInfo->tsSlotId = tsSlotId;
1939

L
Liu Jicong 已提交
1940 1941
  setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo,
                  pTaskInfo);
L
Liu Jicong 已提交
1942 1943
  pOperator->fpSet = createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo,
                                         optrDefaultBufFn, NULL);
1944

1945 1946 1947 1948 1949
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

1950 1951
  return pOperator;

L
Liu Jicong 已提交
1952
_error:
H
Haojun Liao 已提交
1953 1954 1955 1956
  if (pInfo != NULL) {
    destroyStateWindowOperatorInfo(pInfo);
  }

1957 1958
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
1959 1960 1961
  return NULL;
}

1962
void destroySWindowOperatorInfo(void* param) {
1963
  SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
1964 1965 1966
  if (pInfo == NULL) {
    return;
  }
1967

1968
  cleanupBasicInfo(&pInfo->binfo);
H
Haojun Liao 已提交
1969 1970 1971 1972
  colDataDestroy(&pInfo->twAggSup.timeWindowData);

  cleanupAggSup(&pInfo->aggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
D
dapan1121 已提交
1973
  taosMemoryFreeClear(param);
1974 1975
}

H
Haojun Liao 已提交
1976
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode,
1977
                                            SExecTaskInfo* pTaskInfo) {
1978 1979 1980 1981 1982 1983
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1984
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
1985
  initResultSizeInfo(&pOperator->resultInfo, 4096);
1986

1987
  int32_t      numOfCols = 0;
H
Haojun Liao 已提交
1988
  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
H
Haojun Liao 已提交
1989
  SSDataBlock* pResBlock = createDataBlockFromDescNode(pSessionNode->window.node.pOutputDataBlockDesc);
H
Haojun Liao 已提交
1990
  initBasicInfo(&pInfo->binfo, pResBlock);
H
Haojun Liao 已提交
1991

L
Liu Jicong 已提交
1992 1993
  int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
                            pTaskInfo->streamInfo.pState);
1994 1995 1996 1997
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
1998 1999 2000 2001
  pInfo->twAggSup.waterMark = pSessionNode->window.watermark;
  pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType;
  pInfo->gap = pSessionNode->gap;

2002
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2003 2004
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

2005
  pInfo->tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
L
Liu Jicong 已提交
2006 2007 2008
  pInfo->binfo.pRes = pResBlock;
  pInfo->winSup.prevTs = INT64_MIN;
  pInfo->reptScan = false;
H
Haojun Liao 已提交
2009 2010 2011 2012
  code = filterInitFromNode((SNode*)pSessionNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
H
Haojun Liao 已提交
2013

L
Liu Jicong 已提交
2014 2015
  setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED,
                  pInfo, pTaskInfo);
L
Liu Jicong 已提交
2016 2017
  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo,
                                         optrDefaultBufFn, NULL);
2018 2019
  pOperator->pTaskInfo = pTaskInfo;
  code = appendDownstream(pOperator, &downstream, 1);
2020 2021 2022 2023
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2024 2025
  return pOperator;

L
Liu Jicong 已提交
2026
_error:
2027
  destroySWindowOperatorInfo(pInfo);
2028 2029 2030
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
2031
}
5
54liuyao 已提交
2032

5
54liuyao 已提交
2033
void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
2034
                      SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) {
5
54liuyao 已提交
2035 2036
  for (int32_t k = 0; k < numOfOutput; ++k) {
    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
2037 2038 2039 2040 2041
      if (!pTimeWindowData) {
        continue;
      }

      SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);
L
Liu Jicong 已提交
2042 2043
      char*                p = GET_ROWCELL_INTERBUF(pEntryInfo);
      SColumnInfoData      idata = {0};
2044 2045 2046 2047 2048 2049 2050 2051
      idata.info.type = TSDB_DATA_TYPE_BIGINT;
      idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
      idata.pData = p;

      SScalarParam out = {.columnData = &idata};
      SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
      pDestCtx[k].sfp.process(&tw, 1, &out);
      pEntryInfo->numOfRes = 1;
L
Liu Jicong 已提交
2052
    } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
2053
      int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
5
54liuyao 已提交
2054
      if (code != TSDB_CODE_SUCCESS) {
5
54liuyao 已提交
2055
        qError("%s apply combine functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
5
54liuyao 已提交
2056
      }
5
54liuyao 已提交
2057 2058 2059 2060
    } else if (pDestCtx[k].fpSet.combine == NULL) {
      char* funName = fmGetFuncName(pDestCtx[k].functionId);
      qError("%s error, combine funcion for %s is not implemented", GET_TASKID(pTaskInfo), funName);
      taosMemoryFreeClear(funName);
5
54liuyao 已提交
2061 2062 2063 2064
    }
  }
}

dengyihao's avatar
dengyihao 已提交
2065
bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { return streamStateCheck(pState, pKey); }
5
54liuyao 已提交
2066 2067

int32_t setIntervalOutputBuf(SStreamState* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId,
dengyihao's avatar
dengyihao 已提交
2068 2069
                             SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset,
                             SAggSupporter* pAggSup) {
5
54liuyao 已提交
2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
  SWinKey key = {
      .ts = win->skey,
      .groupId = groupId,
  };
  char*   value = NULL;
  int32_t size = pAggSup->resultRowSize;

  if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  *pResult = (SRowBuffPos*)value;
  SResultRow* res = (SResultRow*)((*pResult)->pRowBuff);
  // set time window for current result
dengyihao's avatar
dengyihao 已提交
2083
  res->win = (*win);
5
54liuyao 已提交
2084 2085
  setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset);
  return TSDB_CODE_SUCCESS;
2086 2087
}

2088
static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pUpdatedMap) {
2089 2090 2091 2092
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
  int32_t                      size = taosArrayGetSize(pWinArray);
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
5
54liuyao 已提交
2093
  SExprSupp*                   pSup = &pOperator->exprSupp;
5
54liuyao 已提交
2094 2095 2096
  if (!pInfo->pChildren) {
    return;
  }
5
54liuyao 已提交
2097
  for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
2098 2099 2100 2101
    SWinKey*     pWinRes = taosArrayGet(pWinArray, i);
    SRowBuffPos* pCurResPos = NULL;
    SResultRow*  pCurResult = NULL;
    STimeWindow  parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval);
5
54liuyao 已提交
2102
    if (isDeletedStreamWindow(&parentWin, pWinRes->groupId, pInfo->pState, &pInfo->twAggSup)) {
2103 2104
      continue;
    }
2105

5
54liuyao 已提交
2106
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
2107
    int32_t num = 0;
5
54liuyao 已提交
2108
    for (int32_t j = 0; j < numOfChildren; j++) {
2109 2110 2111 2112
      SOperatorInfo*               pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SStreamIntervalOperatorInfo* pChInfo = pChildOp->info;
      SExprSupp*                   pChildSup = &pChildOp->exprSupp;
      if (!hasIntervalWindow(pChInfo->pState, pWinRes)) {
2113 2114
        continue;
      }
2115
      if (num == 0) {
dengyihao's avatar
dengyihao 已提交
2116 2117
        int32_t code = setIntervalOutputBuf(pInfo->pState, &parentWin, &pCurResPos, pWinRes->groupId, pSup->pCtx,
                                            numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup);
5
54liuyao 已提交
2118
        ASSERT(pCurResPos != NULL);
dengyihao's avatar
dengyihao 已提交
2119
        pCurResult = (SResultRow*)pCurResPos->pRowBuff;
2120
        if (code != TSDB_CODE_SUCCESS || pCurResult == NULL) {
S
Shengliang Guan 已提交
2121
          T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
2122 2123
        }
      }
2124
      num++;
5
54liuyao 已提交
2125 2126
      SRowBuffPos* pChResPos = NULL;
      SResultRow*  pChResult = NULL;
dengyihao's avatar
dengyihao 已提交
2127 2128 2129
      setIntervalOutputBuf(pChInfo->pState, &parentWin, &pChResPos, pWinRes->groupId, pChildSup->pCtx,
                           pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup);
      pChResult = (SResultRow*)pChResPos->pRowBuff;
2130 2131
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true);
      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
2132
    }
2133
    if (num > 0 && pUpdatedMap) {
5
54liuyao 已提交
2134
      saveWinResultInfo(pCurResult->win.skey, pWinRes->groupId, pCurResPos, pUpdatedMap);
2135
      saveOutputBuf(pInfo->pState, pWinRes, pCurResult, pInfo->aggSup.resultRowSize);
2136
    }
5
54liuyao 已提交
2137 2138 2139
  }
}

2140
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) {
L
liuyao 已提交
2141
  if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {
5
54liuyao 已提交
2142
    SWinKey key = {.ts = pWin->skey, .groupId = groupId};
L
liuyao 已提交
2143
    if (!hasIntervalWindow(pState, &key)) {
5
54liuyao 已提交
2144
      return true;
5
54liuyao 已提交
2145
    }
5
54liuyao 已提交
2146
    return false;
5
54liuyao 已提交
2147 2148 2149 2150
  }
  return false;
}

L
Liu Jicong 已提交
2151 2152 2153 2154
int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,
                        STimeWindow* pNextWin) {
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2155 2156 2157 2158
  int32_t prevEndPos = forwardRows - 1 + startPos;
  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
}

H
Haojun Liao 已提交
2159
void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {
5
54liuyao 已提交
2160 2161 2162 2163
  SArray* childIds = taosArrayInit(8, sizeof(int32_t));
  for (int32_t i = 0; i < size; i++) {
    taosArrayPush(childIds, &i);
  }
H
Haojun Liao 已提交
2164
  taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*));
5
54liuyao 已提交
2165 2166 2167 2168
}

static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

2169
static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) {
2170
  tSimpleHashClear(pInfo->aggSup.pResultRowHashTable);
5
54liuyao 已提交
2171
  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
2172
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
L
Liu Jicong 已提交
2173
  pInfo->aggSup.currentPageId = -1;
2174
  streamStateClear(pInfo->pState);
5
54liuyao 已提交
2175 2176
}

5
54liuyao 已提交
2177 2178 2179 2180
static void clearSpecialDataBlock(SSDataBlock* pBlock) {
  if (pBlock->info.rows <= 0) {
    return;
  }
5
54liuyao 已提交
2181 2182 2183
  blockDataCleanup(pBlock);
}

5
54liuyao 已提交
2184 2185 2186 2187 2188 2189
static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
  clearSpecialDataBlock(pBlock);
  int32_t size = taosArrayGetSize(array);
  if (size - (*pIndex) == 0) {
    return;
  }
L
Liu Jicong 已提交
2190
  blockDataEnsureCapacity(pBlock, size - (*pIndex));
2191 2192 2193 2194 2195
  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
5
54liuyao 已提交
2196
  for (; (*pIndex) < size; (*pIndex)++) {
L
Liu Jicong 已提交
2197
    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));
2198 2199 2200 2201 2202
    colDataSetVal(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataSetVal(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
    colDataSetVal(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
    colDataSetVal(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false);
    colDataSetVal(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false);
5
54liuyao 已提交
2203 2204 2205 2206 2207 2208 2209 2210 2211
    pBlock->info.rows++;
  }
  if ((*pIndex) == size) {
    *pIndex = 0;
    taosArrayClear(array);
  }
  blockDataUpdateTsWindow(pBlock, 0);
}

5
54liuyao 已提交
2212
void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SInterval* pInterval) {
5
54liuyao 已提交
2213
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
L
Liu Jicong 已提交
2214
  TSKEY*           tsData = (TSKEY*)pStartCol->pData;
5
54liuyao 已提交
2215 2216
  SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           tsEndData = (TSKEY*)pEndCol->pData;
5
54liuyao 已提交
2217
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
L
Liu Jicong 已提交
2218 2219
  uint64_t*        groupIdData = (uint64_t*)pGroupCol->pData;
  int32_t          chId = getChildIndex(pBlock);
5
54liuyao 已提交
2220
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
5
54liuyao 已提交
2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235
    TSKEY winTs = tsData[i];
    while (winTs < tsEndData[i]) {
      SWinKey winRes = {.ts = winTs, .groupId = groupIdData[i]};
      void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey));
      if (chIds) {
        SArray* chArray = *(SArray**)chIds;
        int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
        if (index != -1) {
          qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId);
          taosArrayRemove(chArray, index);
          if (taosArrayGetSize(chArray) == 0) {
            // pull data is over
            taosArrayDestroy(chArray);
            taosHashRemove(pMap, &winRes, sizeof(SWinKey));
          }
5
54liuyao 已提交
2236 2237
        }
      }
5
54liuyao 已提交
2238
      winTs = taosTimeAdd(winTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
5
54liuyao 已提交
2239 2240 2241
    }
  }
}
5
54liuyao 已提交
2242

2243
static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) {
2244 2245
  int32_t size = taosArrayGetSize(wins);
  for (int32_t i = 0; i < size; i++) {
L
Liu Jicong 已提交
2246
    SWinKey*    winKey = taosArrayGet(wins, i);
2247
    STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
L
liuyao 已提交
2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259
    if (isOverdue(nextWin.ekey, &pInfo->twAggSup) && pInfo->ignoreExpiredData) {
      continue;
    }
    void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
    if (!chIds) {
      SPullWindowInfo pull = {
          .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
      // add pull data request
      if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
        int32_t size1 = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, winKey, size1);
        qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, size1);
2260 2261 2262 2263 2264
      }
    }
  }
}

5
54liuyao 已提交
2265 2266 2267 2268 2269 2270
static void clearFunctionContext(SExprSupp* pSup) {
  for (int32_t i = 0; i < pSup->numOfExprs; i++) {
    pSup->pCtx[i].saveHandle.currentPage = -1;
  }
}

5
54liuyao 已提交
2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351
int32_t getOutputBuf(SStreamState* pState, SRowBuffPos* pPos, SResultRow** pResult) {
  return streamStateGetByPos(pState, pPos, (void**)pResult);
}

int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
                                   SGroupResInfo* pGroupResInfo) {
  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
  SExprInfo*      pExprInfo = pSup->pExprInfo;
  int32_t         numOfExprs = pSup->numOfExprs;
  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;
  SqlFunctionCtx* pCtx = pSup->pCtx;

  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);

  for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {
    SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i);
    SResultRow*  pRow = NULL;
    int32_t      code = getOutputBuf(pState, pPos, &pRow);
    uint64_t     groupId = ((SWinKey*)pPos->pKey)->groupId;
    ASSERT(code == 0);
    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);
    // no results, continue to check the next one
    if (pRow->numOfRows == 0) {
      pGroupResInfo->index += 1;
      continue;
    }
    if (pBlock->info.id.groupId == 0) {
      pBlock->info.id.groupId = groupId;
      void* tbname = NULL;
      if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
        pBlock->info.parTbName[0] = 0;
      } else {
        memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
      }
      streamFreeVal(tbname);
    } else {
      // current value belongs to different group, it can't be packed into one datablock
      if (pBlock->info.id.groupId != groupId) {
        break;
      }
    }

    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
      ASSERT(pBlock->info.rows > 0);
      break;
    }
    pGroupResInfo->index += 1;

    for (int32_t j = 0; j < numOfExprs; ++j) {
      int32_t slotId = pExprInfo[j].base.resSchema.slotId;

      pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);
      SResultRowEntryInfo* pEnryInfo = pCtx[j].resultInfo;
      qDebug("initd:%d, complete:%d, null:%d, res:%d", pEnryInfo->initialized, pEnryInfo->complete,
             pEnryInfo->isNullRes, pEnryInfo->numOfRes);
      if (pCtx[j].fpSet.finalize) {
        int32_t code1 = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
        if (TAOS_FAILED(code1)) {
          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code1));
          T_LONG_JMP(pTaskInfo->env, code1);
        }
      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {
        // do nothing, todo refactor
      } else {
        // expand the result into multiple rows. E.g., _wstart, top(k, 20)
        // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows.
        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
        char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);
        for (int32_t k = 0; k < pRow->numOfRows; ++k) {
          colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);
        }
      }
    }

    pBlock->info.rows += pRow->numOfRows;
  }
  pBlock->info.dataLoad = 1;
  blockDataUpdateTsWindow(pBlock, 0);
  return TSDB_CODE_SUCCESS;
}

2352 2353
void doBuildStreamIntervalResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock,
                                 SGroupResInfo* pGroupResInfo) {
2354 2355 2356 2357 2358 2359 2360 2361 2362 2363
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  // set output datablock version
  pBlock->info.version = pTaskInfo->version;

  blockDataCleanup(pBlock);
  if (!hasRemainResults(pGroupResInfo)) {
    return;
  }

  // clear the existed group id
H
Haojun Liao 已提交
2364
  pBlock->info.id.groupId = 0;
2365
  buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
2366 2367
}

5
54liuyao 已提交
2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378
static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
                                           TSKEY* primaryKeys, int32_t prevPosition) {
  int32_t startPos = prevPosition + 1;
  if (startPos == pDataBlockInfo->rows) {
    startPos = -1;
  } else {
    *pNext = getFinalTimeWindow(primaryKeys[startPos], pInterval);
  }
  return startPos;
}

dengyihao's avatar
dengyihao 已提交
2379
static void setStreamDataVersion(SExecTaskInfo* pTaskInfo, int64_t version, int64_t ckId) {
L
liuyao 已提交
2380
  pTaskInfo->streamInfo.dataVersion = version;
L
liuyao 已提交
2381
  pTaskInfo->streamInfo.checkPointId = ckId;
L
liuyao 已提交
2382 2383
}

2384
static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId,
2385
                                    SSHashObj* pUpdatedMap) {
2386
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;
L
liuyao 已提交
2387
  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
2388 2389 2390 2391 2392 2393 2394

  SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*  pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*      pSup = &pOperatorInfo->exprSupp;
  int32_t         numOfOutput = pSup->numOfExprs;
  int32_t         step = 1;
  TSKEY*          tsCols = NULL;
5
54liuyao 已提交
2395
  SRowBuffPos*    pResPos = NULL;
2396 2397 2398 2399 2400 2401 2402 2403
  SResultRow*     pResult = NULL;
  int32_t         forwardRows = 0;

  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  tsCols = (int64_t*)pColDataInfo->pData;

  int32_t     startPos = 0;
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
5
54liuyao 已提交
2404 2405 2406 2407 2408 2409
  STimeWindow nextWin = {0};
  if (IS_FINAL_OP(pInfo)) {
    nextWin = getFinalTimeWindow(ts, &pInfo->interval);
  } else {
    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
  }
2410 2411 2412 2413 2414 2415 2416 2417 2418 2419
  while (1) {
    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
    if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
      if (startPos < 0) {
        break;
      }
      continue;
    }

L
liuyao 已提交
2420
    if (IS_FINAL_OP(pInfo) && pInfo->pChildren) {
2421 2422 2423 2424 2425 2426
      bool    ignore = true;
      SWinKey winRes = {
          .ts = nextWin.skey,
          .groupId = groupId,
      };
      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
L
liuyao 已提交
2427
      if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && isClosed && !chIds) {
L
Liu Jicong 已提交
2428 2429
        SPullWindowInfo pull = {
            .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
2430
        // add pull data request
5
54liuyao 已提交
2431 2432 2433 2434 2435
        if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
          int32_t size = taosArrayGetSize(pInfo->pChildren);
          addPullWindow(pInfo->pPullDataMap, &winRes, size);
          qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size);
        }
2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458
      } else {
        int32_t index = -1;
        SArray* chArray = NULL;
        int32_t chId = 0;
        if (chIds) {
          chArray = *(void**)chIds;
          chId = getChildIndex(pSDataBlock);
          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
        }
        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {
          ignore = false;
        }
      }

      if (ignore) {
        startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
        if (startPos < 0) {
          break;
        }
        continue;
      }
    }

5
54liuyao 已提交
2459
    int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,
dengyihao's avatar
dengyihao 已提交
2460 2461
                                        pSup->rowEntryInfoOffset, &pInfo->aggSup);
    pResult = (SResultRow*)pResPos->pRowBuff;
2462
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
S
Shengliang Guan 已提交
2463
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
2464
    }
5
54liuyao 已提交
2465 2466 2467 2468 2469 2470
    if (IS_FINAL_OP(pInfo)) {
      forwardRows = 1;
    } else {
      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,
                                             NULL, TSDB_ORDER_ASC);
    }
5
54liuyao 已提交
2471 2472

    SWinKey key = {
dengyihao's avatar
dengyihao 已提交
2473 2474
        .ts = pResult->win.skey,
        .groupId = groupId,
5
54liuyao 已提交
2475
    };
2476
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
5
54liuyao 已提交
2477
      saveWinResult(&key, pResPos, pUpdatedMap);
2478
    }
5
54liuyao 已提交
2479 2480

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
2481
      tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES);
5
54liuyao 已提交
2482
    }
dengyihao's avatar
dengyihao 已提交
2483

2484
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
H
Haojun Liao 已提交
2485
    applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
L
Liu Jicong 已提交
2486
                                    pSDataBlock->info.rows, numOfOutput);
5
54liuyao 已提交
2487
    key.ts = nextWin.skey;
dengyihao's avatar
dengyihao 已提交
2488

2489 2490 2491 2492
    if (pInfo->delKey.ts > key.ts) {
      pInfo->delKey = key;
    }
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505
    if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {
      qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64
             ",maxKey %" PRId64,
             pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
      blockDataUpdateTsWindow(pSDataBlock, 0);

      // timestamp of the data is incorrect
      if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {
        qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,
               pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
      }
    }

5
54liuyao 已提交
2506 2507 2508 2509 2510 2511
    if (IS_FINAL_OP(pInfo)) {
      startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos);
    } else {
      startPos =
          getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);
    }
2512 2513 2514 2515 2516 2517
    if (startPos < 0) {
      break;
    }
  }
}

5
54liuyao 已提交
2518 2519 2520
static inline int winPosCmprImpl(const void* pKey1, const void* pKey2) {
  SRowBuffPos* pos1 = *(SRowBuffPos**)pKey1;
  SRowBuffPos* pos2 = *(SRowBuffPos**)pKey2;
dengyihao's avatar
dengyihao 已提交
2521 2522
  SWinKey*     pWin1 = (SWinKey*)pos1->pKey;
  SWinKey*     pWin2 = (SWinKey*)pos2->pKey;
5
54liuyao 已提交
2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538

  if (pWin1->groupId > pWin2->groupId) {
    return 1;
  } else if (pWin1->groupId < pWin2->groupId) {
    return -1;
  }

  if (pWin1->ts > pWin2->ts) {
    return 1;
  } else if (pWin1->ts < pWin2->ts) {
    return -1;
  }

  return 0;
}

5
54liuyao 已提交
2539
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
2540
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
2541
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
2542 2543
  SOperatorInfo*               downstream = pOperator->pDownstream[0];
  SExprSupp*                   pSup = &pOperator->exprSupp;
2544

5
54liuyao 已提交
2545
  qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
L
Liu Jicong 已提交
2546

5
54liuyao 已提交
2547 2548 2549
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
2550 2551 2552
    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
    if (pInfo->pPullDataRes->info.rows != 0) {
      // process the rest of the data
5
54liuyao 已提交
2553
      printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
2554 2555 2556
      return pInfo->pPullDataRes;
    }

2557
    doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
2558 2559 2560 2561 2562 2563
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
      return pInfo->pDelRes;
    }

2564
    doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
2565 2566 2567
    if (pInfo->binfo.pRes->info.rows != 0) {
      printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
      return pInfo->binfo.pRes;
5
54liuyao 已提交
2568
    }
5
54liuyao 已提交
2569

H
Haojun Liao 已提交
2570
    setOperatorCompleted(pOperator);
5
54liuyao 已提交
2571 2572 2573 2574
    if (!IS_FINAL_OP(pInfo)) {
      clearFunctionContext(&pOperator->exprSupp);
      // semi interval operator clear disk buffer
      clearStreamIntervalOperator(pInfo);
L
liuyao 已提交
2575
      setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId);
5
54liuyao 已提交
2576 2577
      qDebug("===stream===clear semi operator");
    } else {
L
liuyao 已提交
2578
      deleteIntervalDiscBuf(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
L
liuyao 已提交
2579 2580 2581 2582
      if (pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) {
        streamStateCommit(pInfo->pState);
        pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs;
      }
L
liuyao 已提交
2583
      qDebug("===stream===interval final close");
5
54liuyao 已提交
2584 2585
    }
    return NULL;
5
54liuyao 已提交
2586
  } else {
5
54liuyao 已提交
2587
    if (!IS_FINAL_OP(pInfo)) {
2588
      doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
2589 2590 2591 2592 2593
      if (pInfo->pDelRes->info.rows != 0) {
        // process the rest of the data
        printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
        return pInfo->pDelRes;
      }
5
54liuyao 已提交
2594
    }
5
54liuyao 已提交
2595 2596
  }

5
54liuyao 已提交
2597
  if (!pInfo->pUpdated) {
L
liuyao 已提交
2598
    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);
5
54liuyao 已提交
2599 2600 2601
  }
  if (!pInfo->pUpdatedMap) {
    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
L
liuyao 已提交
2602
    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);
5
54liuyao 已提交
2603 2604
  }

5
54liuyao 已提交
2605 2606 2607
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
2608
      pOperator->status = OP_RES_TO_RETURN;
L
Liu Jicong 已提交
2609 2610
      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64,
             IS_FINAL_OP(pInfo) ? "interval final" : "interval semi", pInfo->numOfDatapack);
5
54liuyao 已提交
2611
      pInfo->numOfDatapack = 0;
5
54liuyao 已提交
2612 2613
      break;
    }
5
54liuyao 已提交
2614
    pInfo->numOfDatapack++;
5
54liuyao 已提交
2615
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv");
2616

H
Haojun Liao 已提交
2617
    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
5
54liuyao 已提交
2618
      pInfo->binfo.pRes->info.type = pBlock->info.type;
2619 2620
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
               pBlock->info.type == STREAM_CLEAR) {
2621
      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
5
54liuyao 已提交
2622
      doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap);
2623
      if (IS_FINAL_OP(pInfo)) {
2624 2625 2626
        addRetriveWindow(delWins, pInfo);
        taosArrayAddAll(pInfo->pDelWins, delWins);
        taosArrayDestroy(delWins);
2627 2628
        continue;
      }
5
54liuyao 已提交
2629
      removeResults(delWins, pInfo->pUpdatedMap);
2630 2631
      taosArrayAddAll(pInfo->pDelWins, delWins);
      taosArrayDestroy(delWins);
L
liuyao 已提交
2632 2633 2634 2635 2636 2637 2638 2639

      doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
      if (pInfo->pDelRes->info.rows != 0) {
        // process the rest of the data
        printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
        return pInfo->pDelRes;
      }

2640
      break;
5
54liuyao 已提交
2641
    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
2642
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);
5
54liuyao 已提交
2643
      continue;
5
54liuyao 已提交
2644
    } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
2645 2646
      doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap);
      if (taosArrayGetSize(pInfo->pUpdated) > 0) {
5
54liuyao 已提交
2647 2648 2649
        break;
      }
      continue;
L
Liu Jicong 已提交
2650
    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
2651
      processPullOver(pBlock, pInfo->pPullDataMap, &pInfo->interval);
5
54liuyao 已提交
2652
      continue;
5
54liuyao 已提交
2653 2654 2655 2656
    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
      return pBlock;
    } else {
      ASSERTS(pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
5
54liuyao 已提交
2657
    }
5
54liuyao 已提交
2658

5
54liuyao 已提交
2659 2660 2661 2662
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
2663
    setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
2664 2665 2666 2667
    doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap);
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);
    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);
5
54liuyao 已提交
2668
  }
S
shenglian zhou 已提交
2669

5
54liuyao 已提交
2670
  removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);
5
54liuyao 已提交
2671
  if (IS_FINAL_OP(pInfo)) {
2672
    closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval,
5
54liuyao 已提交
2673
                              pInfo->pPullDataMap, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator);
5
54liuyao 已提交
2674
  }
2675
  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;
5
54liuyao 已提交
2676

2677 2678 2679
  void*   pIte = NULL;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {
5
54liuyao 已提交
2680
    taosArrayPush(pInfo->pUpdated, pIte);
5
54liuyao 已提交
2681
  }
2682
  tSimpleHashCleanup(pInfo->pUpdatedMap);
5
54liuyao 已提交
2683
  pInfo->pUpdatedMap = NULL;
5
54liuyao 已提交
2684
  taosArraySort(pInfo->pUpdated, winPosCmprImpl);
5
54liuyao 已提交
2685

5
54liuyao 已提交
2686 2687
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
  pInfo->pUpdated = NULL;
5
54liuyao 已提交
2688
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
2689 2690 2691 2692

  doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
  if (pInfo->pPullDataRes->info.rows != 0) {
    // process the rest of the data
5
54liuyao 已提交
2693
    printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
2694 2695 2696
    return pInfo->pPullDataRes;
  }

2697
  doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
5
54liuyao 已提交
2698 2699 2700 2701 2702 2703
  if (pInfo->pDelRes->info.rows != 0) {
    // process the rest of the data
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
    return pInfo->pDelRes;
  }

2704
  doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
2705
  if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
2706
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
2707 2708 2709 2710 2711 2712
    return pInfo->binfo.pRes;
  }

  return NULL;
}

5
54liuyao 已提交
2713 2714 2715 2716
int64_t getDeleteMark(SIntervalPhysiNode* pIntervalPhyNode) {
  if (pIntervalPhyNode->window.deleteMark <= 0) {
    return DEAULT_DELETE_MARK;
  }
L
Liu Jicong 已提交
2717
  int64_t deleteMark = TMAX(pIntervalPhyNode->window.deleteMark, pIntervalPhyNode->window.watermark);
5
54liuyao 已提交
2718 2719 2720 2721
  deleteMark = TMAX(deleteMark, pIntervalPhyNode->interval);
  return deleteMark;
}

5
54liuyao 已提交
2722
TSKEY compareTs(void* pKey) {
dengyihao's avatar
dengyihao 已提交
2723
  SWinKey* pWinKey = (SWinKey*)pKey;
5
54liuyao 已提交
2724 2725 2726
  return pWinKey->ts;
}

S
shenglian zhou 已提交
2727 2728
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                     SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
2729 2730 2731
  SIntervalPhysiNode*          pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
2732 2733 2734
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
2735

2736
  pOperator->pTaskInfo = pTaskInfo;
S
shenglian zhou 已提交
2737 2738 2739 2740 2741 2742 2743 2744
  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                .sliding = pIntervalPhyNode->sliding,
                                .intervalUnit = pIntervalPhyNode->intervalUnit,
                                .slidingUnit = pIntervalPhyNode->slidingUnit,
                                .offset = pIntervalPhyNode->offset,
                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pIntervalPhyNode->window.watermark,
5
54liuyao 已提交
2745 2746
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
2747
      .minTs = INT64_MAX,
5
54liuyao 已提交
2748
      .deleteMark = getDeleteMark(pIntervalPhyNode),
L
Liu Jicong 已提交
2749 2750
      .deleteMarkSaved = 0,
      .calTriggerSaved = 0,
L
liuyao 已提交
2751
      .checkPointTs = 0,
dengyihao's avatar
dengyihao 已提交
2752 2753
      .checkPointInterval =
          convertTimePrecision(tsCheckpointInterval, TSDB_TIME_PRECISION_MILLI, pInfo->interval.precision),
S
shenglian zhou 已提交
2754
  };
5
54liuyao 已提交
2755
  ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay");
5
54liuyao 已提交
2756 2757
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
2758
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
2759 2760 2761 2762 2763 2764 2765 2766 2767
  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

S
shenglian zhou 已提交
2768 2769
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
H
Haojun Liao 已提交
2770
  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
5
54liuyao 已提交
2771
  initBasicInfo(&pInfo->binfo, pResBlock);
2772

L
Liu Jicong 已提交
2773 2774
  int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
                            pTaskInfo->streamInfo.pState);
2775 2776 2777 2778
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
2779
  initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs);
2780

5
54liuyao 已提交
2781
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
2782

2783 2784 2785 2786
  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
  *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
  streamStateSetNumber(pInfo->pState, -1);

2787
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
2788 2789
  pInfo->pChildren = NULL;
  if (numOfChild > 0) {
2790
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
5
54liuyao 已提交
2791 2792 2793
    if (!pInfo->pChildren) {
      goto _error;
    }
5
54liuyao 已提交
2794 2795 2796
    for (int32_t i = 0; i < numOfChild; i++) {
      SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp) {
2797
        SStreamIntervalOperatorInfo* pChInfo = pChildOp->info;
2798
        pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
2799
        taosArrayPush(pInfo->pChildren, &pChildOp);
2800
        streamStateSetNumber(pChInfo->pState, i);
5
54liuyao 已提交
2801 2802 2803 2804 2805
        continue;
      }
      goto _error;
    }
  }
5
54liuyao 已提交
2806

2807
  pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
5
54liuyao 已提交
2808

5
54liuyao 已提交
2809 2810 2811 2812
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
    pInfo->isFinal = true;
    pOperator->name = "StreamFinalIntervalOperator";
  } else {
5
54liuyao 已提交
2813
    // semi interval operator does not catch result
5
54liuyao 已提交
2814 2815 2816 2817
    pInfo->isFinal = false;
    pOperator->name = "StreamSemiIntervalOperator";
  }

5
54liuyao 已提交
2818
  if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
5
54liuyao 已提交
2819 2820
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }
5
54liuyao 已提交
2821 2822 2823 2824
  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
  pInfo->pullIndex = 0;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
2825
  pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
5
54liuyao 已提交
2826
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
2827
  pInfo->ignoreExpiredDataSaved = false;
2828
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
2829
  pInfo->delIndex = 0;
H
Haojun Liao 已提交
2830
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
2831 2832
  pInfo->delKey.ts = INT64_MAX;
  pInfo->delKey.groupId = 0;
5
54liuyao 已提交
2833
  pInfo->numOfDatapack = 0;
5
54liuyao 已提交
2834 2835
  pInfo->pUpdated = NULL;
  pInfo->pUpdatedMap = NULL;
dengyihao's avatar
dengyihao 已提交
2836 2837
  pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize,
                                                  compareTs, pInfo->pState, pInfo->twAggSup.deleteMark);
L
liuyao 已提交
2838
  pInfo->dataVersion = 0;
5
54liuyao 已提交
2839

5
54liuyao 已提交
2840
  pOperator->operatorType = pPhyNode->type;
5
54liuyao 已提交
2841 2842 2843 2844
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;

L
Liu Jicong 已提交
2845 2846
  pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo,
                                         optrDefaultBufFn, NULL);
2847
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
5
54liuyao 已提交
2848
    initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
2849
  }
5
54liuyao 已提交
2850 2851 2852 2853 2854 2855 2856 2857
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
2858
  destroyStreamFinalIntervalOperatorInfo(pInfo);
5
54liuyao 已提交
2859 2860 2861
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
5
54liuyao 已提交
2862
}
5
54liuyao 已提交
2863 2864

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
5
54liuyao 已提交
2865
  tSimpleHashCleanup(pSup->pResultRows);
5
54liuyao 已提交
2866 2867
  destroyDiskbasedBuf(pSup->pResultBuf);
  blockDataDestroy(pSup->pScanBlock);
5
54liuyao 已提交
2868 2869
  taosMemoryFreeClear(pSup->pState);
  taosMemoryFreeClear(pSup->pDummyCtx);
5
54liuyao 已提交
2870 2871
}

2872
void destroyStreamSessionAggOperatorInfo(void* param) {
5
54liuyao 已提交
2873
  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;
2874
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
2875
  destroyStreamAggSupporter(&pInfo->streamAggSup);
5
54liuyao 已提交
2876

2877 2878 2879
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
2880 2881
      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);
      destroyOperatorInfo(pChild);
2882
    }
5
54liuyao 已提交
2883
    taosArrayDestroy(pInfo->pChildren);
2884
  }
5
54liuyao 已提交
2885 2886 2887 2888
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  blockDataDestroy(pInfo->pWinBlock);
  blockDataDestroy(pInfo->pUpdateRes);
5
54liuyao 已提交
2889
  tSimpleHashCleanup(pInfo->pStDeleted);
2890

D
dapan1121 已提交
2891
  taosMemoryFreeClear(param);
5
54liuyao 已提交
2892 2893
}

2894 2895
int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,
                        SSDataBlock* pResultBlock) {
H
Haojun Liao 已提交
2896
  initBasicInfo(pBasicInfo, pResultBlock);
2897 2898 2899 2900
  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
  if (code != TSDB_CODE_SUCCESS) {
    return code;
  }
2901

H
Haojun Liao 已提交
2902
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
5
54liuyao 已提交
2903
  for (int32_t i = 0; i < numOfCols; ++i) {
2904
    pSup->pCtx[i].saveHandle.pBuf = NULL;
5
54liuyao 已提交
2905
  }
2906

2907
  ASSERT(numOfCols > 0);
5
54liuyao 已提交
2908 2909 2910 2911 2912 2913
  return TSDB_CODE_SUCCESS;
}

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {
  for (int i = 0; i < nums; i++) {
    pDummy[i].functionId = pCtx[i].functionId;
2914 2915
    pDummy[i].isNotNullFunc = pCtx[i].isNotNullFunc;
    pDummy[i].isPseudoFunc = pCtx[i].isPseudoFunc;
5
54liuyao 已提交
2916 2917
  }
}
5
54liuyao 已提交
2918

5
54liuyao 已提交
2919 2920
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
                    STimeWindowAggSupp* pTwSup) {
2921 2922 2923 2924 2925 2926
  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
    pScanInfo->tsColIndex = tsColIndex;
  }

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
5
54liuyao 已提交
2927
    initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup);
2928 2929
    return;
  }
2930
  SStreamScanInfo* pScanInfo = downstream->info;
5
54liuyao 已提交
2931
  pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
5
54liuyao 已提交
2932
  if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) {
5
54liuyao 已提交
2933
    pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark);
5
54liuyao 已提交
2934
  }
5
54liuyao 已提交
2935
  pScanInfo->twAggSup = *pTwSup;
5
54liuyao 已提交
2936 2937
}

5
54liuyao 已提交
2938 2939 2940 2941 2942 2943 2944 2945 2946 2947
int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap,
                               SStreamState* pState, int32_t keySize, int16_t keyType) {
  pSup->resultRowSize = keySize + getResultRowSize(pCtx, numOfOutput);
  pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR);
  pSup->gap = gap;
  pSup->stateKeySize = keySize;
  pSup->stateKeyType = keyType;
  pSup->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx));
  if (pSup->pDummyCtx == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
2948
  }
H
Haojun Liao 已提交
2949

5
54liuyao 已提交
2950 2951 2952 2953
  initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput);
  pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState));
  *(pSup->pState) = *pState;
  streamStateSetNumber(pSup->pState, -1);
2954

5
54liuyao 已提交
2955 2956
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pSup->pResultRows = tSimpleHashInit(32, hashFn);
X
Xiaoyu Wang 已提交
2957

5
54liuyao 已提交
2958 2959 2960
  int32_t pageSize = 4096;
  while (pageSize < pSup->resultRowSize * 4) {
    pageSize <<= 1u;
5
54liuyao 已提交
2961
  }
5
54liuyao 已提交
2962 2963 2964 2965
  // at least four pages need to be in buffer
  int32_t bufSize = 4096 * 256;
  if (bufSize <= pageSize) {
    bufSize = pageSize * 4;
5
54liuyao 已提交
2966
  }
5
54liuyao 已提交
2967 2968 2969 2970
  if (!osTempSpaceAvailable()) {
    terrno = TSDB_CODE_NO_AVAIL_DISK;
    qError("Init stream agg supporter failed since %s", terrstr(terrno));
    return terrno;
5
54liuyao 已提交
2971
  }
5
54liuyao 已提交
2972 2973 2974
  int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir);
  for (int32_t i = 0; i < numOfOutput; ++i) {
    pCtx[i].saveHandle.pBuf = pSup->pResultBuf;
5
54liuyao 已提交
2975 2976
  }

5
54liuyao 已提交
2977
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
2978
}
5
54liuyao 已提交
2979 2980

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {
5
54liuyao 已提交
2981
  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {
5
54liuyao 已提交
2982 2983 2984 2985 2986
    return true;
  }
  return false;
}

5
54liuyao 已提交
2987 2988
bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) {
  return isInTimeWindow(&pWinInfo->sessionWin.win, ts, gap);
5
54liuyao 已提交
2989 2990
}

5
54liuyao 已提交
2991 2992 2993 2994 2995
void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
                         SSessionKey* pKey) {
  pKey->win.skey = startTs;
  pKey->win.ekey = endTs;
  pKey->groupId = groupId;
2996
  int32_t code = streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey);
5
54liuyao 已提交
2997 2998
  if (code != TSDB_CODE_SUCCESS) {
    SET_SESSION_WIN_KEY_INVALID(pKey);
2999 3000 3001
  }
}

5
54liuyao 已提交
3002
bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->sessionWin.win.skey == 0; }
5
54liuyao 已提交
3003

5
54liuyao 已提交
3004 3005 3006
void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
                         SResultWindowInfo* pCurWin) {
  pCurWin->sessionWin.groupId = groupId;
3007 3008
  pCurWin->sessionWin.win.skey = startTs;
  pCurWin->sessionWin.win.ekey = endTs;
5
54liuyao 已提交
3009
  int32_t size = pAggSup->resultRowSize;
3010 3011
  int32_t code =
      streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, &pCurWin->pOutputBuf, &size);
5
54liuyao 已提交
3012 3013 3014 3015 3016
  if (code == TSDB_CODE_SUCCESS) {
    pCurWin->isOutput = true;
  } else {
    pCurWin->sessionWin.win.skey = startTs;
    pCurWin->sessionWin.win.ekey = endTs;
5
54liuyao 已提交
3017
  }
5
54liuyao 已提交
3018
}
5
54liuyao 已提交
3019

5
54liuyao 已提交
3020 3021
int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) {
  int32_t size = 0;
3022
  int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size);
5
54liuyao 已提交
3023 3024
  if (code != TSDB_CODE_SUCCESS) {
    return code;
5
54liuyao 已提交
3025
  }
5
54liuyao 已提交
3026 3027 3028 3029 3030 3031
  streamStateCurNext(pAggSup->pState, pCur);
  return TSDB_CODE_SUCCESS;
}
void saveDeleteInfo(SArray* pWins, SSessionKey key) {
  // key.win.ekey = key.win.skey;
  taosArrayPush(pWins, &key);
5
54liuyao 已提交
3032 3033
}

5
54liuyao 已提交
3034 3035 3036 3037
void saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) {
  key.win.ekey = key.win.skey;
  tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0);
}
3038

5
54liuyao 已提交
3039 3040 3041 3042 3043
static void removeSessionResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey key) {
  key.win.ekey = key.win.skey;
  tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));
  tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey));
}
5
54liuyao 已提交
3044

5
54liuyao 已提交
3045 3046 3047 3048 3049
static void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey) {
  *pHashKey = *pKey;
  pHashKey->win.ekey = pKey->win.skey;
}

5
54liuyao 已提交
3050 3051 3052
static void removeSessionResults(SSHashObj* pHashMap, SArray* pWins) {
  if (tSimpleHashGetSize(pHashMap) == 0) {
    return;
5
54liuyao 已提交
3053
  }
5
54liuyao 已提交
3054 3055 3056 3057
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
    SSessionKey* pWin = taosArrayGet(pWins, i);
    if (!pWin) continue;
5
54liuyao 已提交
3058 3059
    SSessionKey key = {0};
    getSessionHashKey(pWin, &key);
5
54liuyao 已提交
3060
    tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));
5
54liuyao 已提交
3061 3062 3063
  }
}

dengyihao's avatar
dengyihao 已提交
3064
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId,
5
54liuyao 已提交
3065 3066
                                int32_t rows, int32_t start, int64_t gap, SSHashObj* pResultRows, SSHashObj* pStUpdated,
                                SSHashObj* pStDeleted) {
5
54liuyao 已提交
3067
  for (int32_t i = start; i < rows; ++i) {
3068
    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {
5
54liuyao 已提交
3069 3070
      return i - start;
    }
5
54liuyao 已提交
3071
    if (pWinInfo->sessionWin.win.skey > pStartTs[i]) {
5
54liuyao 已提交
3072
      if (pStDeleted && pWinInfo->isOutput) {
5
54liuyao 已提交
3073
        saveDeleteRes(pStDeleted, pWinInfo->sessionWin);
5
54liuyao 已提交
3074
      }
5
54liuyao 已提交
3075 3076
      removeSessionResult(pStUpdated, pResultRows, pWinInfo->sessionWin);
      pWinInfo->sessionWin.win.skey = pStartTs[i];
5
54liuyao 已提交
3077
    }
5
54liuyao 已提交
3078
    pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pStartTs[i]);
5
54liuyao 已提交
3079
    if (pEndTs) {
5
54liuyao 已提交
3080
      pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pEndTs[i]);
5
54liuyao 已提交
3081 3082 3083 3084 3085
    }
  }
  return rows - start;
}

5
54liuyao 已提交
3086 3087
static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,
                                    int32_t numOfOutput, int32_t* rowEntryInfoOffset) {
3088
  ASSERT(pWinInfo->sessionWin.win.skey <= pWinInfo->sessionWin.win.ekey);
5
54liuyao 已提交
3089
  *pResult = (SResultRow*)pWinInfo->pOutputBuf;
5
54liuyao 已提交
3090
  // set time window for current result
5
54liuyao 已提交
3091
  (*pResult)->win = pWinInfo->sessionWin.win;
3092
  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
5
54liuyao 已提交
3093 3094 3095
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
3096 3097 3098
static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult,
                                  int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput,
                                  SOperatorInfo* pOperator) {
3099
  SExprSupp*     pSup = &pOperator->exprSupp;
3100
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3101
  int32_t        code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
3102
  if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
S
Shengliang Guan 已提交
3103
    return TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
3104
  }
5
54liuyao 已提交
3105
  updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, false);
H
Haojun Liao 已提交
3106
  applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput);
5
54liuyao 已提交
3107 3108 3109
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
3110 3111
static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) {
  streamStateSessionDel(pAggSup->pState, pKey);
5
54liuyao 已提交
3112 3113 3114
  SSessionKey hashKey = {0};
  getSessionHashKey(pKey, &hashKey);
  tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey));
5
54liuyao 已提交
3115 3116 3117 3118 3119 3120 3121 3122 3123 3124
  return true;
}

static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) {
  void* pVal = tSimpleHashGet(pStUpdated, &pWinInfo->sessionWin, sizeof(SSessionKey));
  if (pVal) {
    SResultWindowInfo* pWin = pVal;
    pWinInfo->isOutput = pWin->isOutput;
  }
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
3125 3126
}

5
54liuyao 已提交
3127 3128 3129 3130 3131 3132 3133
SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,
                                       SResultWindowInfo* pNextWin) {
  SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin);
  pNextWin->isOutput = true;
  setSessionWinOutputInfo(pStUpdated, pNextWin);
  int32_t size = 0;
  pNextWin->sessionWin = pCurWin->sessionWin;
3134
  int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size);
5
54liuyao 已提交
3135 3136 3137 3138
  if (code != TSDB_CODE_SUCCESS) {
    SET_SESSION_WIN_INVALID(*pNextWin);
  }
  return pCur;
5
54liuyao 已提交
3139 3140
}

5
54liuyao 已提交
3141 3142 3143 3144 3145 3146 3147 3148 3149
static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated,
                                 SSHashObj* pStDeleted) {
  SExprSupp*                     pSup = &pOperator->exprSupp;
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SResultRow*                    pCurResult = NULL;
  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;
  initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
3150
  // Just look for the window behind StartIndex
5
54liuyao 已提交
3151 3152 3153 3154 3155 3156
  while (1) {
    SResultWindowInfo winInfo = {0};
    SStreamStateCur*  pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo);
    if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) {
      streamStateFreeCur(pCur);
      break;
5
54liuyao 已提交
3157
    }
5
54liuyao 已提交
3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169
    SResultRow* pWinResult = NULL;
    initSessionOutputBuf(&winInfo, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset);
    pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, winInfo.sessionWin.win.ekey);
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->sessionWin.win, true);
    compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
    tSimpleHashRemove(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey));
    if (winInfo.isOutput && pStDeleted) {
      saveDeleteRes(pStDeleted, winInfo.sessionWin);
    }
    removeSessionResult(pStUpdated, pAggSup->pResultRows, winInfo.sessionWin);
    doDeleteSessionWindow(pAggSup, &winInfo.sessionWin);
    streamStateFreeCur(pCur);
5
54liuyao 已提交
3170 3171 3172
  }
}

5
54liuyao 已提交
3173 3174 3175
int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) {
  saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize);
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
3176 3177
}

5
54liuyao 已提交
3178 3179
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated,
                                   SSHashObj* pStDeleted, bool hasEndTs) {
X
Xiaoyu Wang 已提交
3180
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3181
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
3182
  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
H
Haojun Liao 已提交
3183
  uint64_t                       groupId = pSDataBlock->info.id.groupId;
X
Xiaoyu Wang 已提交
3184
  int64_t                        code = TSDB_CODE_SUCCESS;
5
54liuyao 已提交
3185 3186 3187
  SResultRow*                    pResult = NULL;
  int32_t                        rows = pSDataBlock->info.rows;
  int32_t                        winRows = 0;
X
Xiaoyu Wang 已提交
3188

L
liuyao 已提交
3189 3190
  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

5
54liuyao 已提交
3191
  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
3192
  TSKEY*           startTsCols = (int64_t*)pStartTsCol->pData;
5
54liuyao 已提交
3193 3194 3195
  SColumnInfoData* pEndTsCol = NULL;
  if (hasEndTs) {
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);
5
54liuyao 已提交
3196
  } else {
5
54liuyao 已提交
3197
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
3198
  }
X
Xiaoyu Wang 已提交
3199

5
54liuyao 已提交
3200
  TSKEY*               endTsCols = (int64_t*)pEndTsCol->pData;
5
54liuyao 已提交
3201
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
5
54liuyao 已提交
3202
  for (int32_t i = 0; i < rows;) {
5
54liuyao 已提交
3203
    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
3204 3205 3206
      i++;
      continue;
    }
5
54liuyao 已提交
3207 3208 3209 3210 3211
    SResultWindowInfo winInfo = {0};
    setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo);
    setSessionWinOutputInfo(pStUpdated, &winInfo);
    winRows = updateSessionWindowInfo(&winInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap,
                                      pAggSup->pResultRows, pStUpdated, pStDeleted);
5
54liuyao 已提交
3212 3213
    // coverity scan error
    if (!winInfo.pOutputBuf) {
S
Shengliang Guan 已提交
3214
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
5
54liuyao 已提交
3215
    }
L
Liu Jicong 已提交
3216

5
54liuyao 已提交
3217 3218
    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput,
                              pOperator);
5
54liuyao 已提交
3219
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
S
Shengliang Guan 已提交
3220
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
5
54liuyao 已提交
3221
    }
5
54liuyao 已提交
3222 3223
    compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted);
    saveSessionOutputBuf(pAggSup, &winInfo);
5
54liuyao 已提交
3224 3225

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
5
54liuyao 已提交
3226
      code = saveResult(winInfo, pStUpdated);
5
54liuyao 已提交
3227
      if (code != TSDB_CODE_SUCCESS) {
S
Shengliang Guan 已提交
3228
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
5
54liuyao 已提交
3229
      }
5
54liuyao 已提交
3230
    }
5
54liuyao 已提交
3231
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
3232 3233
      SSessionKey key = {0};
      getSessionHashKey(&winInfo.sessionWin, &key);
5
54liuyao 已提交
3234 3235 3236
      tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));
    }

5
54liuyao 已提交
3237 3238 3239 3240
    i += winRows;
  }
}

5
54liuyao 已提交
3241
static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) {
5
54liuyao 已提交
3242
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
3243
  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;
5
54liuyao 已提交
3244
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
3245
  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;
3246
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
3247
  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;
5
54liuyao 已提交
3248
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
5
54liuyao 已提交
3249 3250 3251 3252
    while (1) {
      SSessionKey curWin = {0};
      getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], &curWin);
      if (IS_INVALID_SESSION_WIN_KEY(curWin)) {
3253 3254
        break;
      }
5
54liuyao 已提交
3255 3256 3257 3258
      doDeleteSessionWindow(pAggSup, &curWin);
      if (result) {
        saveDeleteInfo(result, curWin);
      }
3259
    }
5
54liuyao 已提交
3260 3261 3262
  }
}

5
54liuyao 已提交
3263 3264 3265 3266 3267 3268 3269 3270
static inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) {
  SSessionKey* pWin1 = (SSessionKey*)pKey1;
  SSessionKey* pWin2 = (SSessionKey*)pKey2;

  if (pWin1->groupId > pWin2->groupId) {
    return 1;
  } else if (pWin1->groupId < pWin2->groupId) {
    return -1;
5
54liuyao 已提交
3271 3272
  }

5
54liuyao 已提交
3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285
  if (pWin1->win.skey > pWin2->win.skey) {
    return 1;
  } else if (pWin1->win.skey < pWin2->win.skey) {
    return -1;
  }

  return 0;
}

static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) {
  void*   pIte = NULL;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pStUpdated, pIte, &iter)) != NULL) {
3286
    void* key = tSimpleHashGetKey(pIte, NULL);
5
54liuyao 已提交
3287 3288 3289 3290 3291 3292
    taosArrayPush(pUpdated, key);
  }
  taosArraySort(pUpdated, sessionKeyCompareAsc);
  return TSDB_CODE_SUCCESS;
}

3293
void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
5
54liuyao 已提交
3294 3295 3296 3297
  blockDataCleanup(pBlock);
  int32_t size = tSimpleHashGetSize(pStDeleted);
  if (size == 0) {
    return;
3298 3299
  }
  blockDataEnsureCapacity(pBlock, size);
5
54liuyao 已提交
3300 3301 3302 3303 3304
  int32_t iter = 0;
  while (((*Ite) = tSimpleHashIterate(pStDeleted, *Ite, &iter)) != NULL) {
    if (pBlock->info.rows + 1 > pBlock->info.capacity) {
      break;
    }
3305
    SSessionKey*     res = tSimpleHashGetKey(*Ite, NULL);
3306
    SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
3307
    colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)&res->win.skey, false);
3308
    SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
3309
    colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)&res->win.skey, false);
3310
    SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
3311
    colDataSetNULL(pUidCol, pBlock->info.rows);
5
54liuyao 已提交
3312
    SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
3313
    colDataSetVal(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false);
3314
    SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
3315
    colDataSetNULL(pCalStCol, pBlock->info.rows);
3316
    SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
3317
    colDataSetNULL(pCalEdCol, pBlock->info.rows);
3318 3319

    SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
3320 3321 3322

    void* tbname = NULL;
    streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname);
3323
    if (tbname == NULL) {
3324
      colDataSetNULL(pTableCol, pBlock->info.rows);
3325 3326 3327
    } else {
      char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
      STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
3328
      colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
dengyihao's avatar
dengyihao 已提交
3329
      streamFreeVal(tbname);
3330
    }
5
54liuyao 已提交
3331 3332 3333
    pBlock->info.rows += 1;
  }
  if ((*Ite) == NULL) {
5
54liuyao 已提交
3334
    tSimpleHashClear(pStDeleted);
5
54liuyao 已提交
3335 3336 3337
  }
}

5
54liuyao 已提交
3338 3339 3340 3341 3342 3343 3344 3345
static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) {
  SExprSupp*                     pSup = &pOperator->exprSupp;
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
  int32_t                        size = taosArrayGetSize(pWinArray);
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;
  int32_t                        numOfOutput = pSup->numOfExprs;
  int32_t                        numOfChildren = taosArrayGetSize(pInfo->pChildren);
3346

3347
  for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
3348 3349 3350
    SSessionKey*      pWinKey = taosArrayGet(pWinArray, i);
    int32_t           num = 0;
    SResultWindowInfo parentWin = {0};
3351
    for (int32_t j = 0; j < numOfChildren; j++) {
X
Xiaoyu Wang 已提交
3352
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
3353
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
5
54liuyao 已提交
3354
      SStreamAggSupporter*           pChAggSup = &pChInfo->streamAggSup;
5
54liuyao 已提交
3355 3356
      SSessionKey                    chWinKey = {0};
      getSessionHashKey(pWinKey, &chWinKey);
3357 3358 3359
      SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey);
      SResultRow*      pResult = NULL;
      SResultRow*      pChResult = NULL;
5
54liuyao 已提交
3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371
      while (1) {
        SResultWindowInfo childWin = {0};
        childWin.sessionWin = *pWinKey;
        int32_t code = getSessionWinBuf(pChAggSup, pCur, &childWin);
        if (code == TSDB_CODE_SUCCESS && pWinKey->win.skey <= childWin.sessionWin.win.skey &&
            childWin.sessionWin.win.ekey <= pWinKey->win.ekey) {
          if (num == 0) {
            setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin);
            code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);
            if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
              break;
            }
3372
          }
5
54liuyao 已提交
3373 3374 3375 3376 3377 3378 3379 3380
          num++;
          updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, true);
          initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput,
                               pChild->exprSupp.rowEntryInfoOffset);
          compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
          compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL);
          saveResult(parentWin, pStUpdated);
        } else {
5
54liuyao 已提交
3381
          break;
3382 3383
        }
      }
5
54liuyao 已提交
3384 3385 3386 3387
      streamStateFreeCur(pCur);
    }
    if (num > 0) {
      saveSessionOutputBuf(pAggSup, &parentWin);
3388 3389 3390 3391
    }
  }
}

5
54liuyao 已提交
3392 3393 3394 3395 3396 3397 3398 3399 3400 3401
int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed) {
  void*   pIte = NULL;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    SResultWindowInfo* pWinInfo = pIte;
    if (isCloseWindow(&pWinInfo->sessionWin.win, pTwSup)) {
      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {
        int32_t code = saveResult(*pWinInfo, pClosed);
        if (code != TSDB_CODE_SUCCESS) {
          return code;
5
54liuyao 已提交
3402 3403
        }
      }
3404
      SSessionKey* pKey = tSimpleHashGetKey(pIte, NULL);
3405
      tSimpleHashIterateRemove(pHashMap, pKey, sizeof(SSessionKey), &pIte, &iter);
5
54liuyao 已提交
3406 3407 3408 3409 3410
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
3411
static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) {
5
54liuyao 已提交
3412 3413 3414 3415 3416
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);
    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
3417
    closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL);
5
54liuyao 已提交
3418 3419 3420
  }
}

5
54liuyao 已提交
3421 3422 3423 3424
int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) {
  void*   pIte = NULL;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
3425
    SResultWindowInfo* pWinInfo = pIte;
5
54liuyao 已提交
3426
    saveResult(*pWinInfo, pStUpdated);
5
54liuyao 已提交
3427 3428 3429 3430
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
3431
static void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) {
5
54liuyao 已提交
3432 3433
  int32_t size = taosArrayGetSize(pResWins);
  for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
3434 3435
    SSessionKey* pWinKey = taosArrayGet(pResWins, i);
    if (!pWinKey) continue;
5
54liuyao 已提交
3436 3437
    SSessionKey winInfo = {0};
    getSessionHashKey(pWinKey, &winInfo);
5
54liuyao 已提交
3438
    tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0);
3439 3440 3441
  }
}

H
Haojun Liao 已提交
3442
// the allocated memory comes from outer function.
5
54liuyao 已提交
3443 3444 3445
void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) {
  pGroupResInfo->pRows = pArrayList;
  pGroupResInfo->index = 0;
H
Haojun Liao 已提交
3446
  pGroupResInfo->pBuf = NULL;
3447 3448
}

5
54liuyao 已提交
3449 3450 3451 3452 3453 3454 3455 3456
void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo,
                          SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  // set output datablock version
  pBlock->info.version = pTaskInfo->version;

  blockDataCleanup(pBlock);
  if (!hasRemainResults(pGroupResInfo)) {
H
Haojun Liao 已提交
3457
    cleanupGroupResInfo(pGroupResInfo);
3458 3459 3460
    return;
  }

5
54liuyao 已提交
3461
  // clear the existed group id
H
Haojun Liao 已提交
3462
  pBlock->info.id.groupId = 0;
3463
  buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
5
54liuyao 已提交
3464 3465
}

5
54liuyao 已提交
3466
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
5
54liuyao 已提交
3467
  SExprSupp*                     pSup = &pOperator->exprSupp;
5
54liuyao 已提交
3468
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3469
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
5
54liuyao 已提交
3470
  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;
5
54liuyao 已提交
3471 3472 3473
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
3474
    doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
5
54liuyao 已提交
3475
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
3476
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
3477 3478
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
3479 3480 3481 3482
    doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
    if (pBInfo->pRes->info.rows > 0) {
      printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
      return pBInfo->pRes;
5
54liuyao 已提交
3483
    }
5
54liuyao 已提交
3484

H
Haojun Liao 已提交
3485
    setOperatorCompleted(pOperator);
5
54liuyao 已提交
3486
    return NULL;
5
54liuyao 已提交
3487 3488 3489
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
3490 3491 3492 3493
  if (!pInfo->pUpdated) {
    pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey));
  }
  if (!pInfo->pStUpdated) {
3494
    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
5
54liuyao 已提交
3495 3496
    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);
  }
5
54liuyao 已提交
3497 3498 3499 3500 3501
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
3502
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv");
3503

5
54liuyao 已提交
3504 3505 3506
    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
        pBlock->info.type == STREAM_CLEAR) {
      SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));
5
54liuyao 已提交
3507
      // gap must be 0
5
54liuyao 已提交
3508
      doDeleteTimeWindows(pAggSup, pBlock, pWins);
5
54liuyao 已提交
3509
      removeSessionResults(pInfo->pStUpdated, pWins);
5
54liuyao 已提交
3510 3511 3512 3513 3514
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
        // gap must be 0
5
54liuyao 已提交
3515
        doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL);
5
54liuyao 已提交
3516
        rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated);
5
54liuyao 已提交
3517 3518 3519 3520
      }
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
      taosArrayDestroy(pWins);
      continue;
3521
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
3522
      getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated);
5
54liuyao 已提交
3523
      continue;
5
54liuyao 已提交
3524 3525 3526 3527
    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
      return pBlock;
    } else {
      ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
5
54liuyao 已提交
3528
    }
5
54liuyao 已提交
3529

5
54liuyao 已提交
3530 3531 3532 3533
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
3534
    // the pDataBlock are always the same one, no need to call this again
3535
    setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
3536
    doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3537 3538 3539 3540 3541
    if (IS_FINAL_OP(pInfo)) {
      int32_t chIndex = getChildIndex(pBlock);
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
3542 3543
        SOperatorInfo* pChildOp =
            createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
5
54liuyao 已提交
3544
        if (!pChildOp) {
S
Shengliang Guan 已提交
3545
          T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
5
54liuyao 已提交
3546 3547 3548
        }
        taosArrayPush(pInfo->pChildren, &pChildOp);
      }
3549
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
3550
      setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
3551
      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true);
3552
    }
5
54liuyao 已提交
3553 3554
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);
5
54liuyao 已提交
3555 3556 3557
  }
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
3558

5
54liuyao 已提交
3559
  closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated);
5
54liuyao 已提交
3560
  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
5
54liuyao 已提交
3561 3562 3563 3564 3565 3566
  copyUpdateResult(pInfo->pStUpdated, pInfo->pUpdated);
  removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated);
  tSimpleHashCleanup(pInfo->pStUpdated);
  pInfo->pStUpdated = NULL;
  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
  pInfo->pUpdated = NULL;
5
54liuyao 已提交
3567
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
3568

3569 3570 3571 3572 3573 3574
#if 0
  char* pBuf = streamStateSessionDump(pAggSup->pState);
  qDebug("===stream===final session%s", pBuf);
  taosMemoryFree(pBuf);
#endif

3575
  doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
5
54liuyao 已提交
3576
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
3577
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
3578 3579
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
3580 3581 3582 3583 3584 3585 3586

  doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
  if (pBInfo->pRes->info.rows > 0) {
    printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
    return pBInfo->pRes;
  }

H
Haojun Liao 已提交
3587
  setOperatorCompleted(pOperator);
5
54liuyao 已提交
3588
  return NULL;
5
54liuyao 已提交
3589 3590
}

5
54liuyao 已提交
3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                  SExecTaskInfo* pTaskInfo) {
  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
  int32_t                        numOfCols = 0;
  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;
  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

  pOperator->pTaskInfo = pTaskInfo;

  initResultSizeInfo(&pOperator->resultInfo, 4096);
  if (pSessionNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar);
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
5
54liuyao 已提交
3611 3612
    }
  }
5
54liuyao 已提交
3613 3614 3615
  SExprSupp* pSup = &pOperator->exprSupp;

  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
H
Haojun Liao 已提交
3616
  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
5
54liuyao 已提交
3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, pSessionNode->gap,
                                pTaskInfo->streamInfo.pState, 0, 0);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pSessionNode->window.watermark,
      .calTrigger = pSessionNode->window.triggerType,
      .maxTs = INT64_MIN,
      .minTs = INT64_MAX,
  };

  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  if (pSessionNode->window.pTsEnd) {
    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;
  }
  pInfo->binfo.pRes = pResBlock;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pStDeleted = tSimpleHashInit(64, hashFn);
  pInfo->pDelIterator = NULL;
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
  pInfo->pChildren = NULL;
  pInfo->isFinal = false;
  pInfo->pPhyNode = pPhyNode;
  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
3651
  pInfo->ignoreExpiredDataSaved = false;
5
54liuyao 已提交
3652 3653
  pInfo->pUpdated = NULL;
  pInfo->pStUpdated = NULL;
L
liuyao 已提交
3654
  pInfo->dataVersion = 0;
5
54liuyao 已提交
3655

H
Haojun Liao 已提交
3656 3657
  setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
                  OP_NOT_OPENED, pInfo, pTaskInfo);
L
Liu Jicong 已提交
3658 3659
  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo,
                                         optrDefaultBufFn, NULL);
H
Haojun Liao 已提交
3660

5
54liuyao 已提交
3661
  if (downstream) {
5
54liuyao 已提交
3662
    initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
5
54liuyao 已提交
3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679
    code = appendDownstream(pOperator, &downstream, 1);
  }
  return pOperator;

_error:
  if (pInfo != NULL) {
    destroyStreamSessionAggOperatorInfo(pInfo);
  }

  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
  tSimpleHashClear(pInfo->streamAggSup.pResultRows);
  streamStateSessionClear(pInfo->streamAggSup.pState);
5
54liuyao 已提交
3680 3681 3682 3683 3684 3685 3686
}

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
  TSKEY                          maxTs = INT64_MIN;
  SExprSupp*                     pSup = &pOperator->exprSupp;
5
54liuyao 已提交
3687
  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;
3688

5
54liuyao 已提交
3689 3690
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
3691
  }
L
Liu Jicong 已提交
3692

3693
  {
5
54liuyao 已提交
3694
    doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
5
54liuyao 已提交
3695
    if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
3696
      printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
3697 3698 3699
      return pBInfo->pRes;
    }

3700
    doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
3701
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
3702
      printDataBlock(pInfo->pDelRes, "semi session delete");
5
54liuyao 已提交
3703 3704
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
3705

3706
    if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
3707
      clearFunctionContext(&pOperator->exprSupp);
3708 3709
      // semi interval operator clear disk buffer
      clearStreamSessionOperator(pInfo);
H
Haojun Liao 已提交
3710
      setOperatorCompleted(pOperator);
3711 3712
      return NULL;
    }
5
54liuyao 已提交
3713 3714 3715
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
3716 3717 3718 3719
  if (!pInfo->pUpdated) {
    pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey));
  }
  if (!pInfo->pStUpdated) {
3720
    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
5
54liuyao 已提交
3721 3722
    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);
  }
5
54liuyao 已提交
3723 3724 3725
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
3726
      clearSpecialDataBlock(pInfo->pUpdateRes);
3727
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
3728 3729
      break;
    }
H
Haojun Liao 已提交
3730
    printDataBlock(pBlock, "semi session recv");
5
54liuyao 已提交
3731

5
54liuyao 已提交
3732 3733
    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
        pBlock->info.type == STREAM_CLEAR) {
5
54liuyao 已提交
3734
      // gap must be 0
3735
      SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));
5
54liuyao 已提交
3736
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins);
5
54liuyao 已提交
3737
      removeSessionResults(pInfo->pStUpdated, pWins);
5
54liuyao 已提交
3738
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
3739
      taosArrayDestroy(pWins);
5
54liuyao 已提交
3740
      break;
5
54liuyao 已提交
3741
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
3742
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pStUpdated);
5
54liuyao 已提交
3743
      continue;
5
54liuyao 已提交
3744 3745 3746 3747
    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
      return pBlock;
    } else {
      ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
5
54liuyao 已提交
3748 3749
    }

5
54liuyao 已提交
3750 3751 3752 3753
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
5
54liuyao 已提交
3754
    // the pDataBlock are always the same one, no need to call this again
3755
    setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
3756
    doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, NULL, false);
5
54liuyao 已提交
3757 3758 3759 3760
    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
3761
  pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;
3762

5
54liuyao 已提交
3763 3764 3765 3766
  copyUpdateResult(pInfo->pStUpdated, pInfo->pUpdated);
  removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated);
  tSimpleHashCleanup(pInfo->pStUpdated);
  pInfo->pStUpdated = NULL;
5
54liuyao 已提交
3767

5
54liuyao 已提交
3768 3769
  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
  pInfo->pUpdated = NULL;
5
54liuyao 已提交
3770
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
3771

3772 3773 3774 3775 3776 3777
#if 0
  char* pBuf = streamStateSessionDump(pAggSup->pState);
  qDebug("===stream===semi session%s", pBuf);
  taosMemoryFree(pBuf);
#endif

5
54liuyao 已提交
3778
  doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
5
54liuyao 已提交
3779
  if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
3780
    printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
3781 3782 3783
    return pBInfo->pRes;
  }

3784
  doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
3785
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
3786
    printDataBlock(pInfo->pDelRes, "semi session delete");
5
54liuyao 已提交
3787 3788
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
3789

5
54liuyao 已提交
3790 3791 3792
  clearFunctionContext(&pOperator->exprSupp);
  // semi interval operator clear disk buffer
  clearStreamSessionOperator(pInfo);
H
Haojun Liao 已提交
3793
  setOperatorCompleted(pOperator);
5
54liuyao 已提交
3794
  return NULL;
5
54liuyao 已提交
3795
}
3796

3797 3798
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                       SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
3799 3800
  int32_t        code = TSDB_CODE_OUT_OF_MEMORY;
  SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo);
3801 3802 3803
  if (pOperator == NULL) {
    goto _error;
  }
H
Haojun Liao 已提交
3804

3805
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
5
54liuyao 已提交
3806

H
Haojun Liao 已提交
3807
  pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION);
L
Liu Jicong 已提交
3808
  char* name = (pInfo->isFinal) ? "StreamSessionFinalAggOperator" : "StreamSessionSemiAggOperator";
H
Haojun Liao 已提交
3809 3810

  if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
H
Haojun Liao 已提交
3811
    pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
3812
    blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
3813 3814
    pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL,
                                           destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL);
5
54liuyao 已提交
3815
  }
3816

L
Liu Jicong 已提交
3817
  setOperatorInfo(pOperator, name, pPhyNode->type, false, OP_NOT_OPENED, pInfo, pTaskInfo);
H
Haojun Liao 已提交
3818

5
54liuyao 已提交
3819 3820 3821 3822
  pOperator->operatorType = pPhyNode->type;
  if (numOfChild > 0) {
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
    for (int32_t i = 0; i < numOfChild; i++) {
5
54liuyao 已提交
3823 3824
      SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp == NULL) {
5
54liuyao 已提交
3825 3826
        goto _error;
      }
5
54liuyao 已提交
3827 3828 3829 3830
      SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
      pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
      streamStateSetNumber(pChInfo->streamAggSup.pState, i);
      taosArrayPush(pInfo->pChildren, &pChildOp);
3831 3832
    }
  }
3833 3834 3835 3836 3837

  if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }

3838 3839 3840 3841
  return pOperator;

_error:
  if (pInfo != NULL) {
3842
    destroyStreamSessionAggOperatorInfo(pInfo);
3843 3844 3845 3846 3847
  }
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5
54liuyao 已提交
3848

3849
void destroyStreamStateOperatorInfo(void* param) {
X
Xiaoyu Wang 已提交
3850
  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;
3851
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
3852
  destroyStreamAggSupporter(&pInfo->streamAggSup);
5
54liuyao 已提交
3853 3854 3855 3856
  cleanupGroupResInfo(&pInfo->groupResInfo);
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
3857 3858
      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);
      destroyOperatorInfo(pChild);
5
54liuyao 已提交
3859
    }
5
54liuyao 已提交
3860
    taosArrayDestroy(pInfo->pChildren);
5
54liuyao 已提交
3861
  }
5
54liuyao 已提交
3862 3863
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
5
54liuyao 已提交
3864
  tSimpleHashCleanup(pInfo->pSeDeleted);
D
dapan1121 已提交
3865
  taosMemoryFreeClear(param);
5
54liuyao 已提交
3866 3867 3868
}

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {
5
54liuyao 已提交
3869
  if (pWin->winInfo.sessionWin.win.skey <= ts && ts <= pWin->winInfo.sessionWin.win.ekey) {
5
54liuyao 已提交
3870 3871 3872 3873 3874 3875
    return true;
  }
  return false;
}

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {
5
54liuyao 已提交
3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907
  return pKeyData && compareVal(pKeyData, pWin->pStateKey);
}

bool compareStateKey(void* data, void* key) {
  SStateKeys* stateKey = (SStateKeys*)key;
  stateKey->pData = (char*)key + sizeof(SStateKeys);
  return compareVal(data, stateKey);
}

void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
                       SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) {
  int32_t size = pAggSup->resultRowSize;
  pCurWin->winInfo.sessionWin.groupId = groupId;
  pCurWin->winInfo.sessionWin.win.skey = ts;
  pCurWin->winInfo.sessionWin.win.ekey = ts;
  int32_t code =
      streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize,
                                    compareStateKey, &pCurWin->winInfo.pOutputBuf, &size);
  pCurWin->pStateKey =
      (SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize));
  pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
  pCurWin->pStateKey->type = pAggSup->stateKeyType;
  pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);
  pCurWin->pStateKey->isNull = false;

  if (code == TSDB_CODE_SUCCESS) {
    pCurWin->winInfo.isOutput = true;
  } else {
    if (IS_VAR_DATA_TYPE(pAggSup->stateKeyType)) {
      varDataCopy(pCurWin->pStateKey->pData, pKeyData);
    } else {
      memcpy(pCurWin->pStateKey->pData, pKeyData, pCurWin->pStateKey->bytes);
5
54liuyao 已提交
3908 3909 3910
    }
  }

5
54liuyao 已提交
3911 3912 3913 3914 3915 3916
  pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
  pNextWin->winInfo.pOutputBuf = NULL;
  SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
  code = streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0);
  if (code != TSDB_CODE_SUCCESS) {
    SET_SESSION_WIN_INVALID(pNextWin->winInfo);
5
54liuyao 已提交
3917
  }
5
54liuyao 已提交
3918
  streamStateFreeCur(pCur);
5
54liuyao 已提交
3919 3920
}

5
54liuyao 已提交
3921
int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId,
H
Haojun Liao 已提交
3922
                              SColumnInfoData* pKeyCol, int32_t rows, int32_t start, bool* allEqual,
5
54liuyao 已提交
3923
                              SSHashObj* pResultRows, SSHashObj* pSeUpdated, SSHashObj* pSeDeleted) {
5
54liuyao 已提交
3924 3925 3926 3927
  *allEqual = true;
  for (int32_t i = start; i < rows; ++i) {
    char* pKeyData = colDataGetData(pKeyCol, i);
    if (!isTsInWindow(pWinInfo, pTs[i])) {
X
Xiaoyu Wang 已提交
3928
      if (isEqualStateKey(pWinInfo, pKeyData)) {
5
54liuyao 已提交
3929
        if (IS_VALID_SESSION_WIN(pNextWin->winInfo)) {
5
54liuyao 已提交
3930
          // ts belongs to the next window
5
54liuyao 已提交
3931
          if (pTs[i] >= pNextWin->winInfo.sessionWin.win.skey) {
5
54liuyao 已提交
3932 3933 3934 3935 3936 3937 3938
            return i - start;
          }
        }
      } else {
        return i - start;
      }
    }
5
54liuyao 已提交
3939 3940

    if (pWinInfo->winInfo.sessionWin.win.skey > pTs[i]) {
H
Haojun Liao 已提交
3941
      if (pSeDeleted && pWinInfo->winInfo.isOutput) {
5
54liuyao 已提交
3942
        saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin);
5
54liuyao 已提交
3943
      }
5
54liuyao 已提交
3944 3945
      removeSessionResult(pSeUpdated, pResultRows, pWinInfo->winInfo.sessionWin);
      pWinInfo->winInfo.sessionWin.win.skey = pTs[i];
5
54liuyao 已提交
3946
    }
5
54liuyao 已提交
3947
    pWinInfo->winInfo.sessionWin.win.ekey = TMAX(pWinInfo->winInfo.sessionWin.win.ekey, pTs[i]);
5
54liuyao 已提交
3948 3949 3950 3951 3952 3953 3954
    if (!isEqualStateKey(pWinInfo, pKeyData)) {
      *allEqual = false;
    }
  }
  return rows - start;
}

5
54liuyao 已提交
3955 3956
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated,
                                 SSHashObj* pStDeleted) {
X
Xiaoyu Wang 已提交
3957
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3958
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
3959
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
H
Haojun Liao 已提交
3960
  uint64_t                     groupId = pSDataBlock->info.id.groupId;
X
Xiaoyu Wang 已提交
3961 3962 3963 3964
  int64_t                      code = TSDB_CODE_SUCCESS;
  TSKEY*                       tsCols = NULL;
  SResultRow*                  pResult = NULL;
  int32_t                      winRows = 0;
L
liuyao 已提交
3965 3966 3967

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

5
54liuyao 已提交
3968
  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
3969 3970
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
3971
  } else {
X
Xiaoyu Wang 已提交
3972
    return;
5
54liuyao 已提交
3973
  }
L
Liu Jicong 已提交
3974

5
54liuyao 已提交
3975
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
5
54liuyao 已提交
3976 3977
  int32_t              rows = pSDataBlock->info.rows;
  blockDataEnsureCapacity(pAggSup->pScanBlock, rows);
L
Liu Jicong 已提交
3978
  SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
5
54liuyao 已提交
3979
  for (int32_t i = 0; i < rows; i += winRows) {
3980
    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup) || colDataIsNull_s(pKeyColInfo, i)) {
5
54liuyao 已提交
3981 3982 3983
      i++;
      continue;
    }
5
54liuyao 已提交
3984 3985 3986 3987 3988 3989 3990 3991 3992
    char*            pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t          winIndex = 0;
    bool             allEqual = true;
    SStateWindowInfo curWin = {0};
    SStateWindowInfo nextWin = {0};
    setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin);
    setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
    winRows = updateStateWindowInfo(&curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual,
                                    pAggSup->pResultRows, pSeUpdated, pStDeleted);
5
54liuyao 已提交
3993
    if (!allEqual) {
3994
      uint64_t uid = 0;
5
54liuyao 已提交
3995 3996 3997 3998 3999
      appendOneRowToStreamSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
                                       &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
      tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
      doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);
      releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf);
5
54liuyao 已提交
4000 4001
      continue;
    }
5
54liuyao 已提交
4002 4003
    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
                              pOperator);
5
54liuyao 已提交
4004
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
S
Shengliang Guan 已提交
4005
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
5
54liuyao 已提交
4006
    }
5
54liuyao 已提交
4007 4008
    saveSessionOutputBuf(pAggSup, &curWin.winInfo);

5
54liuyao 已提交
4009
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
4010
      code = saveResult(curWin.winInfo, pSeUpdated);
5
54liuyao 已提交
4011
      if (code != TSDB_CODE_SUCCESS) {
S
Shengliang Guan 已提交
4012
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
5
54liuyao 已提交
4013 4014
      }
    }
4015 4016

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
4017 4018
      SSessionKey key = {0};
      getSessionHashKey(&curWin.winInfo.sessionWin, &key);
4019 4020
      tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
    }
5
54liuyao 已提交
4021 4022 4023 4024 4025 4026 4027 4028
  }
}

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

4029
  SExprSupp*                   pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4030
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4031
  SOptrBasicInfo*              pBInfo = &pInfo->binfo;
5
54liuyao 已提交
4032
  if (pOperator->status == OP_RES_TO_RETURN) {
4033
    doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
5
54liuyao 已提交
4034
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4035
      printDataBlock(pInfo->pDelRes, "single state delete");
5
54liuyao 已提交
4036 4037
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
4038 4039 4040 4041 4042

    doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
    if (pBInfo->pRes->info.rows > 0) {
      printDataBlock(pBInfo->pRes, "single state");
      return pBInfo->pRes;
5
54liuyao 已提交
4043
    }
5
54liuyao 已提交
4044

H
Haojun Liao 已提交
4045
    setOperatorCompleted(pOperator);
5
54liuyao 已提交
4046
    return NULL;
5
54liuyao 已提交
4047 4048 4049
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
4050 4051 4052 4053
  if (!pInfo->pUpdated) {
    pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey));
  }
  if (!pInfo->pSeUpdated) {
4054
    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
5
54liuyao 已提交
4055 4056
    pInfo->pSeUpdated = tSimpleHashInit(64, hashFn);
  }
5
54liuyao 已提交
4057 4058 4059 4060 4061
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4062
    printDataBlock(pBlock, "single state recv");
4063

5
54liuyao 已提交
4064 4065 4066 4067
    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
        pBlock->info.type == STREAM_CLEAR) {
      SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins);
5
54liuyao 已提交
4068
      removeSessionResults(pInfo->pSeUpdated, pWins);
5
54liuyao 已提交
4069
      copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
4070 4071
      taosArrayDestroy(pWins);
      continue;
4072
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4073
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated);
5
54liuyao 已提交
4074
      continue;
5
54liuyao 已提交
4075 4076 4077 4078
    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
      return pBlock;
    } else {
      ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
5
54liuyao 已提交
4079
    }
4080

5
54liuyao 已提交
4081 4082 4083 4084
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4085
    // the pDataBlock are always the same one, no need to call this again
4086
    setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4087 4088
    doStreamStateAggImpl(pOperator, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted);
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
4089 4090 4091
  }
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
X
Xiaoyu Wang 已提交
4092

5
54liuyao 已提交
4093 4094 4095 4096 4097
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated);
  copyUpdateResult(pInfo->pSeUpdated, pInfo->pUpdated);
  removeSessionResults(pInfo->pSeDeleted, pInfo->pUpdated);
  tSimpleHashCleanup(pInfo->pSeUpdated);
  pInfo->pSeUpdated = NULL;
5
54liuyao 已提交
4098

5
54liuyao 已提交
4099 4100
  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
  pInfo->pUpdated = NULL;
5
54liuyao 已提交
4101
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
4102

5
54liuyao 已提交
4103 4104 4105 4106 4107 4108
#if 0
  char* pBuf = streamStateSessionDump(pInfo->streamAggSup.pState);
  qDebug("===stream===final session%s", pBuf);
  taosMemoryFree(pBuf);
#endif

4109
  doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
5
54liuyao 已提交
4110
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4111
    printDataBlock(pInfo->pDelRes, "single state delete");
5
54liuyao 已提交
4112 4113 4114
    return pInfo->pDelRes;
  }

5
54liuyao 已提交
4115 4116 4117 4118 4119
  doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
  if (pBInfo->pRes->info.rows > 0) {
    printDataBlock(pBInfo->pRes, "single state");
    return pBInfo->pRes;
  }
H
Haojun Liao 已提交
4120
  setOperatorCompleted(pOperator);
5
54liuyao 已提交
4121
  return NULL;
4122 4123
}

X
Xiaoyu Wang 已提交
4124 4125 4126 4127 4128
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
H
Haojun Liao 已提交
4129
  int32_t                      code = TSDB_CODE_SUCCESS;
5
54liuyao 已提交
4130

X
Xiaoyu Wang 已提交
4131 4132
  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
4133
  if (pInfo == NULL || pOperator == NULL) {
H
Haojun Liao 已提交
4134
    code = TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
4135 4136 4137 4138
    goto _error;
  }

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
4139
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
4140 4141 4142
  if (pStateNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
H
Haojun Liao 已提交
4143
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
5
54liuyao 已提交
4144 4145 4146 4147 4148
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

X
Xiaoyu Wang 已提交
4149 4150
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
5
54liuyao 已提交
4151 4152
      .calTrigger = pStateNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
4153
      .minTs = INT64_MAX,
X
Xiaoyu Wang 已提交
4154
  };
4155

5
54liuyao 已提交
4156
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
4157

5
54liuyao 已提交
4158 4159 4160
  SExprSupp*   pSup = &pOperator->exprSupp;
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);
H
Haojun Liao 已提交
4161
  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
4162
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
4163 4164 4165
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
5
54liuyao 已提交
4166 4167 4168 4169
  int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes;
  int16_t type = pColNode->node.resType.type;
  code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize,
                                type);
5
54liuyao 已提交
4170 4171 4172 4173 4174 4175
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->primaryTsIndex = tsSlotId;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
5
54liuyao 已提交
4176
  pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);
5
54liuyao 已提交
4177
  pInfo->pDelIterator = NULL;
H
Haojun Liao 已提交
4178
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
5
54liuyao 已提交
4179
  pInfo->pChildren = NULL;
5
54liuyao 已提交
4180
  pInfo->ignoreExpiredData = pStateNode->window.igExpired;
4181
  pInfo->ignoreExpiredDataSaved = false;
5
54liuyao 已提交
4182 4183
  pInfo->pUpdated = NULL;
  pInfo->pSeUpdated = NULL;
L
liuyao 已提交
4184
  pInfo->dataVersion = 0;
5
54liuyao 已提交
4185

L
Liu Jicong 已提交
4186 4187
  setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
                  pInfo, pTaskInfo);
L
Liu Jicong 已提交
4188 4189
  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo,
                                         optrDefaultBufFn, NULL);
5
54liuyao 已提交
4190
  initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
5
54liuyao 已提交
4191 4192 4193 4194 4195 4196 4197
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
  return pOperator;

_error:
4198
  destroyStreamStateOperatorInfo(pInfo);
5
54liuyao 已提交
4199 4200 4201 4202
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
4203

4204
void destroyMAIOperatorInfo(void* param) {
4205
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param;
4206
  destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo);
D
dapan1121 已提交
4207
  taosMemoryFreeClear(param);
4208 4209
}

4210
static SResultRow* doSetSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, SAggSupporter* pSup) {
H
Haojun Liao 已提交
4211
  SResultRow* pResult = getNewResultRow(pSup->pResultBuf, &pSup->currentPageId, pSup->resultRowSize);
4212 4213 4214
  if (NULL == pResult) {
    return pResult;
  }
H
Haojun Liao 已提交
4215
  pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
4216 4217
  return pResult;
}
4218

4219 4220 4221 4222 4223 4224 4225 4226
static int32_t setSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, SResultRow** pResult,
                                       SExprSupp* pExprSup, SAggSupporter* pAggSup) {
  if (*pResult == NULL) {
    *pResult = doSetSingleOutputTupleBuf(pResultRowInfo, pAggSup);
    if (*pResult == NULL) {
      return terrno;
    }
  }
4227

4228
  // set time window for current result
4229 4230
  (*pResult)->win = (*win);
  setResultRowInitCtx((*pResult), pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
4231
  return TSDB_CODE_SUCCESS;
4232 4233
}

4234
static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
4235
                                          SSDataBlock* pBlock, SSDataBlock* pResultBlock) {
4236
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
D
dapan1121 已提交
4237
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
4238 4239

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
4240
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
4241
  SInterval*     pInterval = &iaInfo->interval;
4242

5
54liuyao 已提交
4243 4244
  int32_t  startPos = 0;
  int64_t* tsCols = extractTsCol(pBlock, iaInfo);
4245

4246 4247
  TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols);

4248 4249
  // there is an result exists
  if (miaInfo->curTs != INT64_MIN) {
4250
    if (ts != miaInfo->curTs) {
4251
      finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo);
4252
      resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow));
4253
      miaInfo->curTs = ts;
4254
    }
4255 4256
  } else {
    miaInfo->curTs = ts;
4257 4258 4259
  }

  STimeWindow win = {0};
4260
  win.skey = miaInfo->curTs;
4261
  win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
4262

5
54liuyao 已提交
4263
  int32_t ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
4264 4265
  if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) {
    T_LONG_JMP(pTaskInfo->env, ret);
4266 4267
  }

4268 4269
  int32_t currPos = startPos;

4270
  STimeWindow currWin = win;
4271
  while (++currPos < pBlock->info.rows) {
4272
    if (tsCols[currPos] == miaInfo->curTs) {
4273
      continue;
4274 4275 4276
    }

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
H
Haojun Liao 已提交
4277 4278
    applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos,
                                    currPos - startPos, pBlock->info.rows, pSup->numOfExprs);
4279

4280
    finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo);
4281
    resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow));
4282
    miaInfo->curTs = tsCols[currPos];
4283

4284
    currWin.skey = miaInfo->curTs;
4285
    currWin.ekey = taosTimeAdd(currWin.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
4286 4287

    startPos = currPos;
5
54liuyao 已提交
4288
    ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
4289 4290
    if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) {
      T_LONG_JMP(pTaskInfo->env, ret);
4291
    }
4292 4293

    miaInfo->curTs = currWin.skey;
4294
  }
4295

4296
  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
H
Haojun Liao 已提交
4297
  applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
L
Liu Jicong 已提交
4298
                                  pBlock->info.rows, pSup->numOfExprs);
4299 4300
}

4301
static void cleanupAfterGroupResultGen(SMergeAlignedIntervalAggOperatorInfo* pMiaInfo, SSDataBlock* pRes) {
H
Haojun Liao 已提交
4302
  pRes->info.id.groupId = pMiaInfo->groupId;
4303 4304
  pMiaInfo->curTs = INT64_MIN;
  pMiaInfo->groupId = 0;
4305 4306
}

4307
static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
S
shenglian zhou 已提交
4308
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
4309

4310 4311
  SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info;
  SIntervalAggOperatorInfo*             pIaInfo = pMiaInfo->intervalAggOperatorInfo;
4312

4313 4314 4315 4316 4317
  SExprSupp*      pSup = &pOperator->exprSupp;
  SSDataBlock*    pRes = pIaInfo->binfo.pRes;
  SResultRowInfo* pResultRowInfo = &pIaInfo->binfo.resultRowInfo;
  SOperatorInfo*  downstream = pOperator->pDownstream[0];
  int32_t         scanFlag = MAIN_SCAN;
4318

4319 4320
  while (1) {
    SSDataBlock* pBlock = NULL;
4321
    if (pMiaInfo->prefetchedBlock == NULL) {
4322 4323
      pBlock = downstream->fpSet.getNextFn(downstream);
    } else {
4324 4325
      pBlock = pMiaInfo->prefetchedBlock;
      pMiaInfo->prefetchedBlock = NULL;
4326

H
Haojun Liao 已提交
4327
      pMiaInfo->groupId = pBlock->info.id.groupId;
4328
    }
4329

4330
    // no data exists, all query processing is done
4331
    if (pBlock == NULL) {
4332 4333 4334
      // close last unclosed time window
      if (pMiaInfo->curTs != INT64_MIN) {
        finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo);
4335 4336
        resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow));
        cleanupAfterGroupResultGen(pMiaInfo, pRes);
4337
      }
4338

H
Haojun Liao 已提交
4339
      setOperatorCompleted(pOperator);
4340
      break;
4341
    }
4342

H
Haojun Liao 已提交
4343
    if (pMiaInfo->groupId == 0) {
H
Haojun Liao 已提交
4344 4345 4346
      if (pMiaInfo->groupId != pBlock->info.id.groupId) {
        pMiaInfo->groupId = pBlock->info.id.groupId;
        pRes->info.id.groupId = pMiaInfo->groupId;
H
Haojun Liao 已提交
4347 4348
      }
    } else {
H
Haojun Liao 已提交
4349
      if (pMiaInfo->groupId != pBlock->info.id.groupId) {
H
Haojun Liao 已提交
4350
        // if there are unclosed time window, close it firstly.
4351
        ASSERT(pMiaInfo->curTs != INT64_MIN);
H
Haojun Liao 已提交
4352
        finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo);
4353
        resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow));
H
Haojun Liao 已提交
4354

4355 4356
        pMiaInfo->prefetchedBlock = pBlock;
        cleanupAfterGroupResultGen(pMiaInfo, pRes);
H
Haojun Liao 已提交
4357
        break;
5
54liuyao 已提交
4358
      } else {
H
Haojun Liao 已提交
4359
        // continue
H
Haojun Liao 已提交
4360
        pRes->info.id.groupId = pMiaInfo->groupId;
H
Haojun Liao 已提交
4361
      }
4362
    }
4363

4364
    getTableScanInfo(pOperator, &pIaInfo->inputOrder, &scanFlag, false);
4365
    setInputDataBlock(pSup, pBlock, pIaInfo->inputOrder, scanFlag, true);
4366
    doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes);
4367

H
Haojun Liao 已提交
4368
    doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
4369 4370 4371
    if (pRes->info.rows >= pOperator->resultInfo.capacity) {
      break;
    }
4372
  }
4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387
}

static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info;
  SIntervalAggOperatorInfo*             iaInfo = pMiaInfo->intervalAggOperatorInfo;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);

  if (iaInfo->binfo.mergeResultBlock) {
dengyihao's avatar
dengyihao 已提交
4388
    while (1) {
4389
      if (pOperator->status == OP_EXEC_DONE) {
4390 4391
        break;
      }
4392

4393
      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
4394 4395 4396
        break;
      }

4397 4398 4399 4400
      doMergeAlignedIntervalAgg(pOperator);
    }
  } else {
    doMergeAlignedIntervalAgg(pOperator);
4401 4402 4403 4404 4405 4406 4407
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

4408
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode,
4409
                                                      SExecTaskInfo* pTaskInfo) {
4410
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo));
4411
  SOperatorInfo*                        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
4412 4413 4414 4415
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

D
dapan1121 已提交
4416 4417 4418 4419 4420
  miaInfo->intervalAggOperatorInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
  if (miaInfo->intervalAggOperatorInfo == NULL) {
    goto _error;
  }

4421 4422 4423 4424 4425 4426 4427
  SInterval interval = {.interval = pNode->interval,
                        .sliding = pNode->sliding,
                        .intervalUnit = pNode->intervalUnit,
                        .slidingUnit = pNode->slidingUnit,
                        .offset = pNode->offset,
                        .precision = ((SColumnNode*)pNode->window.pTspk)->node.resType.precision};

D
dapan1121 已提交
4428
  SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
4429
  SExprSupp*                pSup = &pOperator->exprSupp;
4430

H
Haojun Liao 已提交
4431 4432 4433 4434 4435
  int32_t code = filterInitFromNode((SNode*)pNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

L
Liu Jicong 已提交
4436 4437 4438 4439
  miaInfo->curTs = INT64_MIN;
  iaInfo->win = pTaskInfo->window;
  iaInfo->inputOrder = TSDB_ORDER_ASC;
  iaInfo->interval = interval;
4440 4441
  iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId;
  iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock;
4442 4443

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
H
Haojun Liao 已提交
4444
  initResultSizeInfo(&pOperator->resultInfo, 512);
4445

H
Haojun Liao 已提交
4446 4447
  int32_t    num = 0;
  SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
H
Haojun Liao 已提交
4448

L
Liu Jicong 已提交
4449 4450
  code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
                    pTaskInfo->streamInfo.pState);
4451 4452 4453
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
4454

H
Haojun Liao 已提交
4455
  SSDataBlock* pResBlock = createDataBlockFromDescNode(pNode->window.node.pOutputDataBlockDesc);
4456
  initBasicInfo(&iaInfo->binfo, pResBlock);
4457
  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
4458

4459
  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, iaInfo);
4460
  if (iaInfo->timeWindowInterpo) {
4461
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
4462 4463
  }

4464
  initResultRowInfo(&iaInfo->binfo.resultRowInfo);
4465
  blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity);
L
Liu Jicong 已提交
4466 4467
  setOperatorInfo(pOperator, "TimeMergeAlignedIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL,
                  false, OP_NOT_OPENED, miaInfo, pTaskInfo);
4468

L
Liu Jicong 已提交
4469 4470
  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo,
                                         optrDefaultBufFn, NULL);
4471 4472 4473 4474 4475 4476 4477 4478 4479

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
4480
  destroyMAIOperatorInfo(miaInfo);
4481 4482 4483 4484
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
4485 4486 4487 4488 4489

//=====================================================================================================================
// merge interval operator
typedef struct SMergeIntervalAggOperatorInfo {
  SIntervalAggOperatorInfo intervalAggOperatorInfo;
L
Liu Jicong 已提交
4490 4491 4492 4493 4494 4495
  SList*                   groupIntervals;
  SListIter                groupIntervalsIter;
  bool                     hasGroupId;
  uint64_t                 groupId;
  SSDataBlock*             prefetchedBlock;
  bool                     inputBlocksFinished;
4496 4497
} SMergeIntervalAggOperatorInfo;

S
slzhou 已提交
4498
typedef struct SGroupTimeWindow {
L
Liu Jicong 已提交
4499
  uint64_t    groupId;
S
slzhou 已提交
4500 4501 4502
  STimeWindow window;
} SGroupTimeWindow;

4503
void destroyMergeIntervalOperatorInfo(void* param) {
4504
  SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
S
slzhou 已提交
4505
  tdListFree(miaInfo->groupIntervals);
4506
  destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo);
4507

D
dapan1121 已提交
4508
  taosMemoryFreeClear(param);
4509 4510
}

L
Liu Jicong 已提交
4511 4512
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win,
                                    SSDataBlock* pResultBlock) {
4513 4514 4515
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
4516
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
4517 4518 4519
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId);
L
Liu Jicong 已提交
4520 4521
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(
      iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
4522
  ASSERT(p1 != NULL);
5
54liuyao 已提交
4523
  //  finalizeResultRows(iaInfo->aggSup.pResultBuf, p1, pResultBlock, pTaskInfo);
4524
  tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
4525 4526 4527
  return TSDB_CODE_SUCCESS;
}

4528 4529 4530 4531
static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock,
                                        STimeWindow* newWin) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
4532
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
4533

S
slzhou 已提交
4534 4535
  SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin};
  tdListAppend(miaInfo->groupIntervals, &groupTimeWindow);
4536

S
slzhou 已提交
4537 4538 4539 4540 4541
  SListIter iter = {0};
  tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD);
  SListNode* listNode = NULL;
  while ((listNode = tdListNext(&iter)) != NULL) {
    SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
L
Liu Jicong 已提交
4542
    if (prevGrpWin->groupId != tableGroupId) {
S
slzhou 已提交
4543 4544
      continue;
    }
4545

S
slzhou 已提交
4546
    STimeWindow* prevWin = &prevGrpWin->window;
H
Haojun Liao 已提交
4547
    if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) {
5
54liuyao 已提交
4548
      //      finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
S
slzhou 已提交
4549 4550
      tdListPopNode(miaInfo->groupIntervals, listNode);
    }
4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566
  }

  return 0;
}

static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                                   int32_t scanFlag, SSDataBlock* pResultBlock) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*     pExprSup = &pOperatorInfo->exprSupp;

  int32_t     startPos = 0;
  int32_t     numOfOutput = pExprSup->numOfExprs;
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
H
Haojun Liao 已提交
4567
  uint64_t    tableGroupId = pBlock->info.id.groupId;
4568
  bool        ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
4569 4570 4571
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

4572 4573
  STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval,
                                        iaInfo->inputOrder);
4574 4575 4576 4577 4578

  int32_t ret =
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                             numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
S
Shengliang Guan 已提交
4579
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
4580 4581 4582 4583
  }

  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
4584
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
4585
  ASSERT(forwardRows > 0);
4586 4587 4588

  // prev time window not interpolation yet.
  if (iaInfo->timeWindowInterpo) {
4589
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
4590 4591 4592 4593 4594 4595
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);

    // restore current time window
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                                 numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS) {
S
Shengliang Guan 已提交
4596
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
4597 4598 4599 4600 4601 4602 4603
    }

    // window start key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup);
  }

  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true);
H
Haojun Liao 已提交
4604
  applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
L
Liu Jicong 已提交
4605
                                  pBlock->info.rows, numOfOutput);
4606 4607 4608 4609 4610 4611 4612 4613
  doCloseWindow(pResultRowInfo, iaInfo, pResult);

  // output previous interval results after this interval (&win) is closed
  outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win);

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = forwardRows - 1 + startPos;
4614 4615
    startPos =
        getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->inputOrder);
4616 4617 4618 4619 4620 4621 4622 4623 4624
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
    int32_t code =
        setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                               pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
S
Shengliang Guan 已提交
4625
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
4626 4627 4628 4629
    }

    ekey = ascScan ? nextWin.ekey : nextWin.skey;
    forwardRows =
4630
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
4631 4632 4633 4634 4635

    // window start(end) key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup);

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true);
H
Haojun Liao 已提交
4636
    applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
L
Liu Jicong 已提交
4637
                                    pBlock->info.rows, numOfOutput);
4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672
    doCloseWindow(pResultRowInfo, iaInfo, pResult);

    // output previous interval results after this interval (&nextWin) is closed
    outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin);
  }

  if (iaInfo->timeWindowInterpo) {
    saveDataBlockLastRow(iaInfo->pPrevValues, pBlock, iaInfo->pInterpCols);
  }
}

static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExprSupp*                     pExpSupp = &pOperator->exprSupp;

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);

  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
H
Haojun Liao 已提交
4673
        miaInfo->groupId = pBlock->info.id.groupId;
4674 4675 4676 4677
        miaInfo->prefetchedBlock = NULL;
      }

      if (pBlock == NULL) {
S
slzhou 已提交
4678
        tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD);
4679 4680 4681 4682 4683 4684
        miaInfo->inputBlocksFinished = true;
        break;
      }

      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
H
Haojun Liao 已提交
4685 4686
        miaInfo->groupId = pBlock->info.id.groupId;
      } else if (miaInfo->groupId != pBlock->info.id.groupId) {
4687 4688 4689 4690
        miaInfo->prefetchedBlock = pBlock;
        break;
      }

4691
      getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag, false);
4692
      setInputDataBlock(pExpSupp, pBlock, iaInfo->inputOrder, scanFlag, true);
4693 4694 4695 4696 4697 4698 4699
      doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);

      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
        break;
      }
    }

H
Haojun Liao 已提交
4700
    pRes->info.id.groupId = miaInfo->groupId;
4701 4702 4703
  }

  if (miaInfo->inputBlocksFinished) {
S
slzhou 已提交
4704
    SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter);
4705

S
slzhou 已提交
4706 4707
    if (listNode != NULL) {
      SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data);
5
54liuyao 已提交
4708
      //      finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
H
Haojun Liao 已提交
4709
      pRes->info.id.groupId = grpWin->groupId;
4710 4711 4712 4713
    }
  }

  if (pRes->info.rows == 0) {
H
Haojun Liao 已提交
4714
    setOperatorCompleted(pOperator);
4715 4716 4717 4718 4719 4720 4721
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

4722 4723 4724
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode,
                                               SExecTaskInfo* pTaskInfo) {
  SMergeIntervalAggOperatorInfo* pMergeIntervalInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
4725
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
4726
  if (pMergeIntervalInfo == NULL || pOperator == NULL) {
4727 4728 4729
    goto _error;
  }

5
54liuyao 已提交
4730 4731
  int32_t    num = 0;
  SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
4732 4733 4734 4735 4736 4737 4738

  SInterval interval = {.interval = pIntervalPhyNode->interval,
                        .sliding = pIntervalPhyNode->sliding,
                        .intervalUnit = pIntervalPhyNode->intervalUnit,
                        .slidingUnit = pIntervalPhyNode->slidingUnit,
                        .offset = pIntervalPhyNode->offset,
                        .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
4739

4740
  pMergeIntervalInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow));
4741

4742
  SIntervalAggOperatorInfo* pIntervalInfo = &pMergeIntervalInfo->intervalAggOperatorInfo;
L
Liu Jicong 已提交
4743
  pIntervalInfo->win = pTaskInfo->window;
4744
  pIntervalInfo->inputOrder = TSDB_ORDER_ASC;
L
Liu Jicong 已提交
4745
  pIntervalInfo->interval = interval;
4746 4747
  pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock;
  pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
4748 4749 4750 4751

  SExprSupp* pExprSupp = &pOperator->exprSupp;

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
4752
  initResultSizeInfo(&pOperator->resultInfo, 4096);
4753

L
Liu Jicong 已提交
4754 4755
  int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
                            pTaskInfo->streamInfo.pState);
4756 4757 4758
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
4759

H
Haojun Liao 已提交
4760
  SSDataBlock* pResBlock = createDataBlockFromDescNode(pIntervalPhyNode->window.node.pOutputDataBlockDesc);
4761 4762
  initBasicInfo(&pIntervalInfo->binfo, pResBlock);
  initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win);
4763

4764 4765
  pIntervalInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, num, pIntervalInfo);
  if (pIntervalInfo->timeWindowInterpo) {
4766
    pIntervalInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
4767
    if (pIntervalInfo->binfo.resultRowInfo.openWindow == NULL) {
4768 4769 4770 4771
      goto _error;
    }
  }

4772
  initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo);
L
Liu Jicong 已提交
4773 4774
  setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false,
                  OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo);
L
Liu Jicong 已提交
4775 4776
  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo,
                                         optrDefaultBufFn, NULL);
4777 4778 4779 4780 4781 4782 4783 4784 4785

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
H
Haojun Liao 已提交
4786 4787 4788 4789
  if (pMergeIntervalInfo != NULL) {
    destroyMergeIntervalOperatorInfo(pMergeIntervalInfo);
  }

4790 4791 4792
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
4793
}
4794 4795 4796

static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
4797 4798
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
  SExprSupp*                   pSup = &pOperator->exprSupp;
4799 4800 4801 4802 4803 4804

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
4805
    doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
4806
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4807
      printDataBlock(pInfo->pDelRes, "single interval delete");
4808 4809 4810
      return pInfo->pDelRes;
    }

4811
    doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
4812 4813 4814
    if (pInfo->binfo.pRes->info.rows > 0) {
      printDataBlock(pInfo->binfo.pRes, "single interval");
      return pInfo->binfo.pRes;
4815
    }
L
liuyao 已提交
4816
    deleteIntervalDiscBuf(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
H
Haojun Liao 已提交
4817
    setOperatorCompleted(pOperator);
L
liuyao 已提交
4818 4819
    if (pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) {
      streamStateCommit(pInfo->pState);
L
liuyao 已提交
4820
      setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId);
L
liuyao 已提交
4821 4822
      pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs;
    }
5
54liuyao 已提交
4823
    return NULL;
4824 4825 4826 4827
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];

5
54liuyao 已提交
4828
  if (!pInfo->pUpdated) {
L
liuyao 已提交
4829
    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);
5
54liuyao 已提交
4830 4831 4832
  }
  if (!pInfo->pUpdatedMap) {
    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
L
liuyao 已提交
4833
    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);
5
54liuyao 已提交
4834 4835
  }

4836 4837 4838
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
4839 4840
      qDebug("===stream===return data:single interval. recv datablock num:%" PRIu64, pInfo->numOfDatapack);
      pInfo->numOfDatapack = 0;
4841 4842
      break;
    }
5
54liuyao 已提交
4843
    pInfo->numOfDatapack++;
4844 4845
    printDataBlock(pBlock, "single interval recv");

4846 4847
    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
        pBlock->info.type == STREAM_CLEAR) {
5
54liuyao 已提交
4848
      doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap);
4849 4850
      continue;
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4851
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);
4852
      continue;
5
54liuyao 已提交
4853
    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
5
54liuyao 已提交
4854
      printDataBlock(pBlock, "single interval");
5
54liuyao 已提交
4855 4856 4857
      return pBlock;
    } else {
      ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872
    }

    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {
      // set input version
      pTaskInfo->version = pBlock->info.version;
    }

    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }

    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
    // caller. Note that all the time window are not close till now.
    // the pDataBlock are always the same one, no need to call this again
4873
    setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
4874 4875 4876 4877
    if (pInfo->invertible) {
      setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type);
    }

5
54liuyao 已提交
4878
    doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap);
5
54liuyao 已提交
4879 4880
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);
4881 4882
  }
  pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
4883
  removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);
4884 4885
  closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL,
                            pInfo->pUpdatedMap, pInfo->pDelWins, pOperator);
4886

4887 4888 4889
  void*   pIte = NULL;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {
5
54liuyao 已提交
4890
    taosArrayPush(pInfo->pUpdated, pIte);
4891
  }
5
54liuyao 已提交
4892
  taosArraySort(pInfo->pUpdated, winPosCmprImpl);
4893

5
54liuyao 已提交
4894 4895
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
  pInfo->pUpdated = NULL;
4896
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
4897
  tSimpleHashCleanup(pInfo->pUpdatedMap);
5
54liuyao 已提交
4898
  pInfo->pUpdatedMap = NULL;
5
54liuyao 已提交
4899

5
54liuyao 已提交
4900 4901 4902 4903 4904 4905
#if 0
  char* pBuf = streamStateIntervalDump(pInfo->pState);
  qDebug("===stream===interval state%s", pBuf);
  taosMemoryFree(pBuf);
#endif

4906
  doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
4907
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4908
    printDataBlock(pInfo->pDelRes, "single interval delete");
4909 4910 4911
    return pInfo->pDelRes;
  }

4912
  doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
4913 4914 4915 4916 4917 4918
  if (pInfo->binfo.pRes->info.rows > 0) {
    printDataBlock(pInfo->binfo.pRes, "single interval");
    return pInfo->binfo.pRes;
  }

  return NULL;
4919 4920 4921 4922 4923
}

SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
4924
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
4925 4926 4927 4928 4929
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

H
Haojun Liao 已提交
4930
  int32_t    code = TSDB_CODE_SUCCESS;
4931 4932
  int32_t    numOfCols = 0;
  SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
H
Haojun Liao 已提交
4933

H
Haojun Liao 已提交
4934
  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
dengyihao's avatar
dengyihao 已提交
4935 4936 4937 4938 4939 4940 4941
  pInfo->interval = (SInterval){
      .interval = pIntervalPhyNode->interval,
      .sliding = pIntervalPhyNode->sliding,
      .intervalUnit = pIntervalPhyNode->intervalUnit,
      .slidingUnit = pIntervalPhyNode->slidingUnit,
      .offset = pIntervalPhyNode->offset,
      .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,
4942
  };
H
Haojun Liao 已提交
4943

dengyihao's avatar
dengyihao 已提交
4944
  pInfo->twAggSup = (STimeWindowAggSupp){
4945 4946 4947
      .waterMark = pIntervalPhyNode->window.watermark,
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
4948
      .minTs = INT64_MAX,
5
54liuyao 已提交
4949
      .deleteMark = getDeleteMark(pIntervalPhyNode),
L
liuyao 已提交
4950
      .checkPointTs = 0,
dengyihao's avatar
dengyihao 已提交
4951 4952
      .checkPointInterval =
          convertTimePrecision(tsCheckpointInterval, TSDB_TIME_PRECISION_MILLI, pInfo->interval.precision),
4953
  };
H
Haojun Liao 已提交
4954

L
liuyao 已提交
4955
  ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay");
4956

4957 4958
  pOperator->pTaskInfo = pTaskInfo;
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
4959
  pInfo->ignoreExpiredDataSaved = false;
4960 4961 4962
  pInfo->isFinal = false;

  SExprSupp* pSup = &pOperator->exprSupp;
H
Haojun Liao 已提交
4963 4964 4965 4966
  initBasicInfo(&pInfo->binfo, pResBlock);
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

4967
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
4968
  initResultSizeInfo(&pOperator->resultInfo, 4096);
4969

4970
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
L
Liu Jicong 已提交
4971 4972
  code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
                    pTaskInfo->streamInfo.pState);
4973 4974 4975 4976
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
4977 4978 4979 4980 4981 4982 4983 4984
  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
4985 4986

  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
4987
  pInfo->invertible = false;
4988 4989 4990 4991 4992
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
  pInfo->delIndex = 0;
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
  initResultRowInfo(&pInfo->binfo.resultRowInfo);

4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005
  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
  *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
  streamStateSetNumber(pInfo->pState, -1);

  pInfo->pPhyNode = NULL;  // create new child
  pInfo->pPullDataMap = NULL;
  pInfo->pPullWins = NULL;  // SPullWindowInfo
  pInfo->pullIndex = 0;
  pInfo->pPullDataRes = NULL;
  pInfo->isFinal = false;
  pInfo->pChildren = NULL;
  pInfo->delKey.ts = INT64_MAX;
  pInfo->delKey.groupId = 0;
5
54liuyao 已提交
5006
  pInfo->numOfDatapack = 0;
5
54liuyao 已提交
5007 5008
  pInfo->pUpdated = NULL;
  pInfo->pUpdatedMap = NULL;
dengyihao's avatar
dengyihao 已提交
5009 5010
  pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize,
                                                  compareTs, pInfo->pState, pInfo->twAggSup.deleteMark);
5011

L
Liu Jicong 已提交
5012 5013
  setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,
                  pInfo, pTaskInfo);
5014 5015
  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL,
                                         destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL);
5016

5
54liuyao 已提交
5017
  initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
5018 5019 5020 5021 5022 5023 5024 5025
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5026
  destroyStreamFinalIntervalOperatorInfo(pInfo);
5027 5028 5029 5030
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}