timewindowoperator.c 227.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15
#include "executorimpl.h"
X
Xiaoyu Wang 已提交
16
#include "function.h"
5
54liuyao 已提交
17
#include "functionMgt.h"
H
Haojun Liao 已提交
18
#include "tcommon.h"
5
54liuyao 已提交
19
#include "tcompare.h"
L
Liu Jicong 已提交
20
#include "tdatablock.h"
H
Haojun Liao 已提交
21
#include "tfill.h"
22
#include "ttime.h"
23 24 25 26 27 28

typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;

5
54liuyao 已提交
29 30
#define IS_FINAL_OP(op) ((op)->isFinal)

5
54liuyao 已提交
31 32
typedef struct SPullWindowInfo {
  STimeWindow window;
L
Liu Jicong 已提交
33
  uint64_t    groupId;
5
54liuyao 已提交
34 35
} SPullWindowInfo;

36 37
typedef struct SOpenWindowInfo {
  SResultRowPosition pos;
L
Liu Jicong 已提交
38
  uint64_t           groupId;
39 40
} SOpenWindowInfo;

5
54liuyao 已提交
41
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
5
54liuyao 已提交
42

43 44
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);

L
Liu Jicong 已提交
45 46
static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult,
                                              uint64_t groupId);
47 48
static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult);

H
Haojun Liao 已提交
49 50 51 52 53 54 55 56 57
///*
// * There are two cases to handle:
// *
// * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including
// * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey.
// * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be
// *    merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there
// *    is a previous result generated or not.
// */
58
// static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) {
H
Haojun Liao 已提交
59 60
//  // do nothing
//}
61

X
Xiaoyu Wang 已提交
62
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }
63 64 65

static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
66
                                      int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
67 68 69 70 71 72 73 74 75 76 77
                                      SExecTaskInfo* pTaskInfo) {
  SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                  masterscan, tableGroupId, pTaskInfo, true, pAggSup);

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // set time window for current result
  pResultRow->win = (*win);
78

79
  *pResult = pResultRow;
80
  setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
81

82 83 84 85 86 87 88 89 90 91 92 93 94
  return TSDB_CODE_SUCCESS;
}

static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) {
  int64_t* ts = (int64_t*)pColData->pData;
  int32_t  delta = includeEndpoint ? 1 : 0;

  int64_t duration = pWin->ekey - pWin->skey + delta;
  ts[2] = duration;            // set the duration
  ts[3] = pWin->skey;          // window start key
  ts[4] = pWin->ekey + delta;  // window end key
}

95
static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) {
96 97 98
  pRowSup->win.ekey = ts;
  pRowSup->prevTs = ts;
  pRowSup->numOfRows += 1;
99
  pRowSup->groupId = groupId;
100 101
}

dengyihao's avatar
dengyihao 已提交
102 103
static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex,
                                     uint64_t groupId) {
104 105 106
  pRowSup->startRowIndex = rowIndex;
  pRowSup->numOfRows = 0;
  pRowSup->win.skey = tsList[rowIndex];
107
  pRowSup->groupId = groupId;
108 109 110 111
}

static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
                                                   int16_t pos, int16_t order, int64_t* pData) {
112
  int32_t forwardRows = 0;
113 114 115 116

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
117
      forwardRows = end;
118 119

      if (pData[end + pos] == ekey) {
120
        forwardRows += 1;
121 122 123
      }
    }
  } else {
124
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
125
    if (end >= 0) {
126
      forwardRows = end;
127

128
      if (pData[end + pos] == ekey) {
129
        forwardRows += 1;
130 131
      }
    }
X
Xiaoyu Wang 已提交
132 133 134 135 136 137 138 139
    //    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
    //    if (end >= 0) {
    //      forwardRows = pos - end;
    //
    //      if (pData[end] == ekey) {
    //        forwardRows += 1;
    //      }
    //    }
140 141
  }

142 143
  assert(forwardRows >= 0);
  return forwardRows;
144 145
}

5
54liuyao 已提交
146
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY*  keyList = (TSKEY*)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
163 164 165 166 167 168 169 170 171 172 173
      if (key >= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key < keyList[lastPos]) {
        lastPos += 1;
        if (lastPos >= num) {
          return -1;
        } else {
          return lastPos;
        }
      }
174 175 176 177 178 179

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        firstPos = midPos + 1;
180 181
      } else if (key > keyList[midPos]) {
        lastPos = midPos - 1;
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

X
Xiaoyu Wang 已提交
217 218
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
                                 __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num = -1;
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);

  if (order == TSDB_ORDER_ASC) {
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
      if (item != NULL) {
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
240
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
241 242
      }
    } else {
243
      num = pDataBlockInfo->rows - startPos;
244
      if (item != NULL) {
245
        item->lastKey = pDataBlockInfo->window.ekey + step;
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
      }
    }
  }

  assert(num >= 0);
  return num;
}

static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
    return;
  }

  int64_t key = tw->skey, interval = pInterval->interval;
  // convert key to second
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;

  if (pInterval->intervalUnit == 'y') {
    interval *= 12;
  }

  struct tm tm;
  time_t    t = (time_t)key;
  taosLocalTime(&t, &tm);

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
277
  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
278 279 280 281

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
282
  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
283 284 285 286

  tw->ekey -= 1;
}

287 288
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs,
                               int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
289
  SqlFunctionCtx* pCtx = pSup->pCtx;
290

291
  int32_t index = 1;
292
  for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
H
Haojun Liao 已提交
293
    if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
294 295 296 297
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

X
Xiaoyu Wang 已提交
298
    SFunctParam*     pParam = &pCtx[k].param[0];
299 300
    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId);

301
    ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey);
302

303
    double v1 = 0, v2 = 0, v = 0;
304
    if (prevRowIndex == -1) {
305
      SGroupKeys* p = taosArrayGet(pPrevValues, index);
306
      GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData);
307
    } else {
308
      GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex));
309 310
    }

311
    GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
312

313
#if 0
314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            //            pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
333 334
#endif

X
Xiaoyu Wang 已提交
335 336 337
    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
    SPoint point = (SPoint){.key = windowKey, .val = &v};
338

X
Xiaoyu Wang 已提交
339
    taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
340

X
Xiaoyu Wang 已提交
341 342 343 344 345 346
    if (type == RESULT_ROW_START_INTERP) {
      pCtx[k].start.key = point.key;
      pCtx[k].start.val = v;
    } else {
      pCtx[k].end.key = point.key;
      pCtx[k].end.val = v;
347
    }
X
Xiaoyu Wang 已提交
348 349 350

    index += 1;
  }
351
#if 0
352
  }
353
#endif
354 355 356 357 358 359 360 361 362 363 364 365 366 367
}

static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

368 369
static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock,
                                              const TSKEY* tsCols, STimeWindow* win, SExprSupp* pSup) {
370
  bool ascQuery = (pInfo->inputOrder == TSDB_ORDER_ASC);
371

372
  TSKEY curTs = tsCols[pos];
373 374

  SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
X
Xiaoyu Wang 已提交
375
  TSKEY       lastTs = *(int64_t*)pTsKey->pData;
376 377 378 379 380

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery ? win->skey : win->ekey;
  if (key == curTs) {
381
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
382 383 384
    return true;
  }

385 386
  // it is the first time window, no need to do interpolation
  if (pTsKey->isNull && pos == 0) {
387
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
388 389
  } else {
    TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]);
390 391
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key,
                              RESULT_ROW_START_INTERP, pSup);
392 393 394 395 396
  }

  return true;
}

397 398 399
static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex,
                                            SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey,
                                            STimeWindow* win) {
400
  int32_t order = pInfo->inputOrder;
401 402

  TSKEY actualEndKey = tsCols[endRowIndex];
403
  TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey;
404 405

  // not ended in current data block, do not invoke interpolation
406
  if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) {
407
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
408 409 410
    return false;
  }

411
  // there is actual end point of current time window, no interpolation needs
412
  if (key == actualEndKey) {
413
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
414 415 416
    return true;
  }

417
  int32_t nextRowIndex = endRowIndex + 1;
418 419 420
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
421 422
  doTimeWindowInterpolation(pInfo->pPrevValues, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key,
                            RESULT_ROW_END_INTERP, pSup);
423 424 425
  return true;
}

5
54liuyao 已提交
426
bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo) {
427 428
  if (pInterval->interval != pInterval->sliding &&
      (pWin->ekey < pBlockInfo->calWin.skey || pWin->skey > pBlockInfo->calWin.ekey)) {
5
54liuyao 已提交
429 430 431 432 433
    return false;
  }
  return true;
}

434
static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
5
54liuyao 已提交
435
                                      TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
X
Xiaoyu Wang 已提交
436
  bool ascQuery = (order == TSDB_ORDER_ASC);
437 438 439 440 441 442 443 444 445 446

  int32_t precision = pInterval->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
    return -1;
  }

5
54liuyao 已提交
447 448 449 450
  if (!inSlidingWindow(pInterval, pNext, pDataBlockInfo) && order == TSDB_ORDER_ASC) {
    return -1;
  }

451
  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
452 453 454 455
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
456
    startPos = prevPosition + 1;
457
  } else {
458
    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
459 460
      startPos = 0;
    } else {
461
      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
    }
  }

  /* interp query with fill should not skip time window */
  //  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
  //    return startPos;
  //  }

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
    if (ascQuery) {
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
      }
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
      }
    }
  }

  return startPos;
}

505 506
static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
  ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp == true;
  }
}

static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp = true;
  }
}

523 524
static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult,
                                        STimeWindow* win, int32_t startPos, int32_t forwardRows, SExprSupp* pSup) {
525
  if (!pInfo->timeWindowInterpo) {
526 527 528
    return;
  }

529
  ASSERT(pBlock != NULL);
530 531 532 533 534
  if (pBlock->pDataBlock == NULL) {
    //    tscError("pBlock->pDataBlock == NULL");
    return;
  }

535
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
536 537

  TSKEY* tsCols = (TSKEY*)(pColInfo->pData);
538
  bool   done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
539
  if (!done) {  // it is not interpolated, now start to generated the interpolated value
540
    bool interp = setTimeWindowInterpolationStartTs(pInfo, startPos, pBlock, tsCols, win, pSup);
541 542 543 544
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
545
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
546 547 548 549 550 551 552 553
  }

  // point interpolation does not require the end key time window interpolation.
  //  if (pointInterpQuery) {
  //    return;
  //  }

  // interpolation query does not generate the time window end interpolation
554
  done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
555
  if (!done) {
556
    int32_t endRowIndex = startPos + forwardRows - 1;
557

558
    TSKEY endKey = (pInfo->inputOrder == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
559
    bool  interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
560 561 562 563
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
564
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
565 566 567
  }
}

568 569
static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) {
  if (pBlock->pDataBlock == NULL) {
570 571 572
    return;
  }

573 574 575 576 577 578 579
  size_t num = taosArrayGetSize(pPrevKeys);
  for (int32_t k = 0; k < num; ++k) {
    SColumn* pc = taosArrayGet(pCols, k);

    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId);

    SGroupKeys* pkey = taosArrayGet(pPrevKeys, k);
X
Xiaoyu Wang 已提交
580
    for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) {
581 582 583 584 585 586 587 588 589 590 591 592 593 594
      if (colDataIsNull_s(pColInfo, i)) {
        continue;
      }

      char* val = colDataGetData(pColInfo, i);
      if (IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, varDataTLen(val));
        ASSERT(varDataTLen(val) <= pkey->bytes);
      } else {
        memcpy(pkey->pData, val, pkey->bytes);
      }

      break;
    }
595 596 597
  }
}

598 599 600 601
static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo,
                                       SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) {
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

602
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
603
  SExprSupp*                pSup = &pOperatorInfo->exprSupp;
604

L
Liu Jicong 已提交
605 606
  int32_t startPos = 0;
  int32_t numOfOutput = pSup->numOfExprs;
607

608
  SResultRow* pResult = NULL;
609

610
  while (1) {
L
Liu Jicong 已提交
611 612 613
    SListNode*          pn = tdListGetHead(pResultRowInfo->openWindow);
    SOpenWindowInfo*    pOpenWin = (SOpenWindowInfo*)pn->data;
    uint64_t            groupId = pOpenWin->groupId;
614
    SResultRowPosition* p1 = &pOpenWin->pos;
615 616 617
    if (p->pageId == p1->pageId && p->offset == p1->offset) {
      break;
    }
618

619
    SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1, false);
620
    ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
621

622
    if (pr->closed) {
X
Xiaoyu Wang 已提交
623 624
      ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
             isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
625 626
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
627 628
      continue;
    }
629

630
    STimeWindow w = pr->win;
631 632
    int32_t     ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx,
                                             numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
633
    if (ret != TSDB_CODE_SUCCESS) {
634
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
635 636 637 638
    }

    ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP));

X
Xiaoyu Wang 已提交
639 640
    SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
    int64_t     prevTs = *(int64_t*)pTsKey->pData;
641 642 643 644
    if (groupId == pBlock->info.groupId) {
      doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey,
                                RESULT_ROW_END_INTERP, pSup);
    }
645 646

    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
647
    setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
648

649
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, true);
H
Haojun Liao 已提交
650 651
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, pBlock->info.rows,
                     numOfExprs);
652 653 654

    if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
      closeResultRow(pr);
655 656
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
X
Xiaoyu Wang 已提交
657
    } else {  // the remains are can not be closed yet.
658
      break;
659
    }
660
  }
661
}
662

5
54liuyao 已提交
663
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
5
54liuyao 已提交
664
  if (!pBlock || pBlock->info.rows == 0) {
5
54liuyao 已提交
665
    qDebug("===stream===printDataBlock: Block is Null or Empty");
5
54liuyao 已提交
666 667
    return;
  }
L
Liu Jicong 已提交
668
  char* pBuf = NULL;
5
54liuyao 已提交
669 670
  qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
  taosMemoryFree(pBuf);
5
54liuyao 已提交
671 672
}

5
54liuyao 已提交
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);

int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;
      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;
      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;
      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

      if (comparefn(pKey, keyList, lastPos) > 0) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

5
54liuyao 已提交
729
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
730

X
Xiaoyu Wang 已提交
731 732 733
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;
5
54liuyao 已提交
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (key >= getValuefn(keyList, lastPos)) return lastPos;
      if (key == getValuefn(keyList, firstPos)) return firstPos;
      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (key <= getValuefn(keyList, firstPos)) return firstPos;
      if (key == getValuefn(keyList, lastPos)) return lastPos;

      if (key > getValuefn(keyList, lastPos)) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
780 781
  }

5
54liuyao 已提交
782 783 784
  return midPos;
}

5
54liuyao 已提交
785
int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {
L
Liu Jicong 已提交
786
  SArray*          res = (SArray*)data;
5
54liuyao 已提交
787
  SPullWindowInfo* pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
788
  SPullWindowInfo* pData = (SPullWindowInfo*)pKey;
5
54liuyao 已提交
789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819
  if (pData->window.skey == pos->window.skey) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->window.skey > pos->window.skey) {
    return 1;
  }
  return -1;
}

static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {
  int32_t size = taosArrayGetSize(pPullWins);
  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);
  if (index == -1) {
    index = 0;
  } else {
    if (comparePullWinKey(pPullInfo, pPullWins, index) > 0) {
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
820 821 822
int32_t compareResKey(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
  SResKeyPos* pos = taosArrayGetP(res, index);
H
Haojun Liao 已提交
823
  SWinKey*    pData = (SWinKey*)pKey;
5
54liuyao 已提交
824 825 826 827 828 829 830 831 832 833 834 835 836
  if (pData->ts == *(int64_t*)pos->key) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->ts > *(int64_t*)pos->key) {
    return 1;
  }
  return -1;
}

837
static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SArray* pUpdated) {
5
54liuyao 已提交
838
  int32_t size = taosArrayGetSize(pUpdated);
H
Haojun Liao 已提交
839
  SWinKey data = {.ts = ts, .groupId = groupId};
5
54liuyao 已提交
840
  int32_t index = binarySearchCom(pUpdated, size, &data, TSDB_ORDER_DESC, compareResKey);
5
54liuyao 已提交
841 842 843
  if (index == -1) {
    index = 0;
  } else {
5
54liuyao 已提交
844
    if (compareResKey(&data, pUpdated, index) > 0) {
5
54liuyao 已提交
845 846 847 848 849
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
H
Haojun Liao 已提交
850

5
54liuyao 已提交
851 852 853 854 855
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
5
54liuyao 已提交
856 857
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
X
Xiaoyu Wang 已提交
858
  if (taosArrayInsert(pUpdated, index, &newPos) == NULL) {
5
54liuyao 已提交
859 860 861 862 863
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
864 865 866 867 868 869 870 871
static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SHashObj* pUpdatedMap) {
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
H
Haojun Liao 已提交
872 873
  SWinKey key = {.ts = ts, .groupId = groupId};
  if (taosHashPut(pUpdatedMap, &key, sizeof(SWinKey), &newPos, sizeof(void*)) != TSDB_CODE_SUCCESS) {
5
54liuyao 已提交
874 875 876
    taosMemoryFree(newPos);
  }
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
877 878
}

5
54liuyao 已提交
879
static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj* pUpdatedMap) {
dengyihao's avatar
dengyihao 已提交
880
  return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap);
5
54liuyao 已提交
881 882 883 884
}

static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
  return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated);
5
54liuyao 已提交
885 886
}

5
54liuyao 已提交
887
static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) {
5
54liuyao 已提交
888 889
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
H
Haojun Liao 已提交
890 891
    SWinKey* pW = taosArrayGet(pWins, i);
    taosHashRemove(pUpdatedMap, pW, sizeof(SWinKey));
5
54liuyao 已提交
892 893 894
  }
}

895
int64_t getWinReskey(void* data, int32_t index) {
896
  SArray*  res = (SArray*)data;
H
Haojun Liao 已提交
897
  SWinKey* pos = taosArrayGet(res, index);
898 899 900
  return pos->ts;
}

5
54liuyao 已提交
901 902
int32_t compareWinRes(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
903
  SWinKey*    pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
904
  SResKeyPos* pData = (SResKeyPos*)pKey;
5
54liuyao 已提交
905 906 907 908 909 910 911 912 913 914 915 916 917
  if (*(int64_t*)pData->key == pos->ts) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (*(int64_t*)pData->key > pos->ts) {
    return 1;
  }
  return -1;
}

5
54liuyao 已提交
918
static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) {
L
Liu Jicong 已提交
919 920
  int32_t delSize = taosArrayGetSize(pDelWins);
  if (taosHashGetSize(pUpdatedMap) == 0 || delSize == 0) {
5
54liuyao 已提交
921
    return;
dengyihao's avatar
dengyihao 已提交
922
  }
L
Liu Jicong 已提交
923
  void* pIte = NULL;
5
54liuyao 已提交
924
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
925
    SResKeyPos* pResKey = *(SResKeyPos**)pIte;
5
54liuyao 已提交
926 927
    int32_t     index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes);
    if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) {
928
      taosArrayRemove(pDelWins, index);
929
      delSize = taosArrayGetSize(pDelWins);
930 931 932 933
    }
  }
}

5
54liuyao 已提交
934 935 936 937 938
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) {
  ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
  return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark;
}

L
Liu Jicong 已提交
939
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); }
5
54liuyao 已提交
940

5
54liuyao 已提交
941
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
942
                            int32_t scanFlag) {
943
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
944

945
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
946
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
947

X
Xiaoyu Wang 已提交
948
  int32_t     startPos = 0;
949
  int32_t     numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
950 951
  int64_t*    tsCols = extractTsCol(pBlock, pInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
952
  bool        ascScan = (pInfo->inputOrder == TSDB_ORDER_ASC);
X
Xiaoyu Wang 已提交
953 954
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;
955

956 957
  STimeWindow win =
      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder);
958 959
  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
960 961
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
962
  }
X
Xiaoyu Wang 已提交
963 964
  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
965
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
966
  ASSERT(forwardRows > 0);
967 968

  // prev time window not interpolation yet.
969
  if (pInfo->timeWindowInterpo) {
970
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
971
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);
972 973

    // restore current time window
974 975
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
976
    if (ret != TSDB_CODE_SUCCESS) {
977
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
978 979
    }

980
    // window start key interpolation
981
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
982
  }
983

984 985
  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
986
                   numOfOutput);
987 988

  doCloseWindow(pResultRowInfo, pInfo, pResult);
989 990 991

  STimeWindow nextWin = win;
  while (1) {
992
    int32_t prevEndPos = forwardRows - 1 + startPos;
993
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->inputOrder);
994 995 996 997
    if (startPos < 0) {
      break;
    }
    // null data, failed to allocate more memory buffer
X
Xiaoyu Wang 已提交
998
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
999
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1000
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
1001
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
1002 1003
    }

X
Xiaoyu Wang 已提交
1004
    ekey = ascScan ? nextWin.ekey : nextWin.skey;
1005
    forwardRows =
1006
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
1007
    // window start(end) key interpolation
1008
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
L
Liu Jicong 已提交
1009
    // TODO: add to open window? how to close the open windows after input blocks exhausted?
S
shenglian zhou 已提交
1010
#if 0
1011 1012 1013 1014
    if ((ascScan && ekey <= pBlock->info.window.ekey) ||
        (!ascScan && ekey >= pBlock->info.window.skey)) {
      // window start(end) key interpolation
      doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
1015
    } else if (pInfo->timeWindowInterpo) {
1016 1017
      addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
    }
S
shenglian zhou 已提交
1018
#endif
1019
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
H
Haojun Liao 已提交
1020 1021
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
1022
    doCloseWindow(pResultRowInfo, pInfo, pResult);
1023 1024 1025
  }

  if (pInfo->timeWindowInterpo) {
1026
    saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
1027
  }
1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
}

void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
  // current result is done in computing final results.
  if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
    closeResultRow(pResult);
    tdListPopHead(pResultRowInfo->openWindow);
  }
}

1038 1039 1040 1041 1042
SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, uint64_t groupId) {
  SOpenWindowInfo openWin = {0};
  openWin.pos.pageId = pResult->pageId;
  openWin.pos.offset = pResult->offset;
  openWin.groupId = groupId;
L
Liu Jicong 已提交
1043
  SListNode* pn = tdListGetTail(pResultRowInfo->openWindow);
1044
  if (pn == NULL) {
1045 1046
    tdListAppend(pResultRowInfo->openWindow, &openWin);
    return openWin.pos;
1047 1048
  }

L
Liu Jicong 已提交
1049
  SOpenWindowInfo* px = (SOpenWindowInfo*)pn->data;
1050 1051
  if (px->pos.pageId != openWin.pos.pageId || px->pos.offset != openWin.pos.offset || px->groupId != openWin.groupId) {
    tdListAppend(pResultRowInfo->openWindow, &openWin);
1052 1053
  }

1054
  return openWin.pos;
1055 1056 1057 1058
}

int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
  TSKEY* tsCols = NULL;
1059

1060 1061 1062 1063
  if (pBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;

1064 1065 1066 1067 1068 1069
    // no data in primary ts
    if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) {
      return NULL;
    }

    if (tsCols[0] != 0 && (pBlock->info.window.skey == 0 && pBlock->info.window.ekey == 0)) {
1070 1071 1072 1073 1074
      blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
    }
  }

  return tsCols;
1075 1076 1077 1078 1079 1080 1081
}

static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
  }

L
Liu Jicong 已提交
1082
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1083
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
1084
  SExprSupp*                pSup = &pOperator->exprSupp;
1085

1086 1087
  int32_t scanFlag = MAIN_SCAN;

X
Xiaoyu Wang 已提交
1088
  int64_t        st = taosGetTimestampUs();
1089 1090 1091
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1092
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1093 1094 1095 1096
    if (pBlock == NULL) {
      break;
    }

1097
    getTableScanInfo(pOperator, &pInfo->inputOrder, &scanFlag);
1098

1099
    if (pInfo->scalarSupp.pExprInfo != NULL) {
L
Liu Jicong 已提交
1100 1101
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
1102 1103
    }

1104
    // the pDataBlock are always the same one, no need to call this again
1105
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, scanFlag, true);
1106 1107
    blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);

1108
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag);
1109 1110
  }

1111
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->resultTsOrder);
1112
  OPTR_SET_OPENED(pOperator);
1113 1114

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1115 1116 1117
  return TSDB_CODE_SUCCESS;
}

1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
static bool compareVal(const char* v, const SStateKeys* pKey) {
  if (IS_VAR_DATA_TYPE(pKey->type)) {
    if (varDataLen(v) != varDataLen(pKey->pData)) {
      return false;
    } else {
      return strncmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0;
    }
  } else {
    return memcmp(pKey->pData, v, pKey->bytes) == 0;
  }
}

1130
static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
L
Liu Jicong 已提交
1131
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1132
  SExprSupp*     pSup = &pOperator->exprSupp;
1133

1134
  SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
1135 1136 1137
  int64_t          gid = pBlock->info.groupId;

  bool    masterScan = true;
1138
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1139 1140
  int16_t bytes = pStateColInfoData->info.bytes;

1141
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1142 1143 1144 1145 1146
  TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

1147
  struct SColumnDataAgg* pAgg = NULL;
1148
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
X
Xiaoyu Wang 已提交
1149
    pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
1150
    if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
1151 1152 1153 1154 1155
      continue;
    }

    char* val = colDataGetData(pStateColInfoData, j);

1156
    if (gid != pRowSup->groupId || !pInfo->hasKey) {
1157 1158 1159 1160 1161 1162 1163
      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }

1164 1165
      pInfo->hasKey = true;

1166 1167
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1168
    } else if (compareVal(val, &pInfo->stateKey)) {
1169
      doKeepTuple(pRowSup, tsList[j], gid);
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // a new state window started
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1180 1181
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1182
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1183
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1184 1185 1186
      }

      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1187 1188
      doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                       pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1189 1190

      // here we start a new session window
1191 1192
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1193 1194 1195 1196 1197 1198 1199

      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }
1200 1201 1202 1203 1204
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1205 1206
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1207
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1208
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1209 1210 1211
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
H
Haojun Liao 已提交
1212 1213
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
                   pBlock->info.rows, numOfOutput);
1214 1215
}

1216
static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
1217 1218 1219 1220 1221
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
1222

1223 1224
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  SExprSupp*     pSup = &pOperator->exprSupp;
1225

1226
  SOptrBasicInfo* pBInfo = &pInfo->binfo;
1227 1228

  if (pOperator->status == OP_RES_TO_RETURN) {
1229
    while (1) {
1230
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1231
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1232

1233
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }

      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1244
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1245 1246
  }

1247
  int32_t order = TSDB_ORDER_ASC;
1248
  int64_t st = taosGetTimestampUs();
1249 1250 1251

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  while (1) {
1252
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1253 1254 1255 1256
    if (pBlock == NULL) {
      break;
    }

1257
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1258 1259
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1260 1261 1262
    doStateWindowAggImpl(pOperator, pInfo, pBlock);
  }

X
Xiaoyu Wang 已提交
1263
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1264 1265
  pOperator->status = OP_RES_TO_RETURN;

1266
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1267
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1268
  while (1) {
1269
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1270
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1271

1272
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1273 1274 1275 1276
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
1277

1278 1279 1280 1281 1282
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1283
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1284 1285
}

1286
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
1287
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
1288
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1289 1290 1291 1292 1293 1294 1295 1296

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pBlock = pInfo->binfo.pRes;

  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
1297
    return pOperator->fpSet.getStreamResFn(pOperator);
1298 1299 1300 1301 1302 1303 1304
  } else {
    pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
    if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
      return NULL;
    }

    blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
1305 1306
    while (1) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1307
      doFilter(pInfo->pCondition, pBlock, NULL);
1308

1309
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1310 1311 1312 1313
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
1314

1315 1316 1317
      if (pBlock->info.rows > 0) {
        break;
      }
1318 1319
    }

1320 1321 1322
    size_t rows = pBlock->info.rows;
    pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1323
    return (rows == 0) ? NULL : pBlock;
1324 1325 1326 1327
  }
}

// todo merged with the build group result.
1328
static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
1329
                                  int32_t* rowEntryInfoOffset) {
1330 1331 1332 1333 1334 1335 1336
  size_t num = taosArrayGetSize(pUpdateList);

  for (int32_t i = 0; i < num; ++i) {
    SResKeyPos* pPos = taosArrayGetP(pUpdateList, i);

    SFilePage*  bufPage = getBufPage(pBuf, pPos->pos.pageId);
    SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
1337

1338
    for (int32_t j = 0; j < numOfOutput; ++j) {
1339
      SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, j, rowEntryInfoOffset);
1340 1341
      if (pRow->numOfRows < pEntry->numOfRes) {
        pRow->numOfRows = pEntry->numOfRes;
1342 1343 1344 1345 1346 1347
      }
    }

    releaseBufPage(pBuf, bufPage);
  }
}
5
54liuyao 已提交
1348
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
L
Liu Jicong 已提交
1349
  for (int i = 0; i < num; i++) {
5
54liuyao 已提交
1350 1351
    if (type == STREAM_INVERT) {
      fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
L
Liu Jicong 已提交
1352
    } else if (type == STREAM_NORMAL) {
5
54liuyao 已提交
1353 1354 1355 1356
      fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
    }
  }
}
5
54liuyao 已提交
1357

1358
void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) {
1359
  SResultRow*     pResult = getResultRowByPos(pResultBuf, p1, false);
1360
  SqlFunctionCtx* pCtx = pSup->pCtx;
5
54liuyao 已提交
1361
  for (int32_t i = 0; i < numOfOutput; ++i) {
1362
    pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
1363 1364 1365 1366 1367 1368 1369 1370 1371
    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
      continue;
    }
    pResInfo->initialized = false;
    if (pCtx[i].functionId != -1) {
      pCtx[i].fpSet.init(&pCtx[i], pResInfo);
    }
  }
5
54liuyao 已提交
1372 1373 1374
  SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pResultBuf, bufPage);
5
54liuyao 已提交
1375 1376
}

5
54liuyao 已提交
1377
bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
X
Xiaoyu Wang 已提交
1378
                   int32_t numOfOutput) {
1379
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId);
5
54liuyao 已提交
1380
  SResultRowPosition* p1 =
1381
      (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1382 1383
  if (!p1) {
    // window has been closed
5
54liuyao 已提交
1384
    return false;
1385
  }
1386
  doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput);
5
54liuyao 已提交
1387
  return true;
5
54liuyao 已提交
1388 1389
}

1390 1391 1392 1393
bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) {
  size_t bytes = sizeof(TSKEY);
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId);
  SResultRowPosition* p1 =
1394
      (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1395 1396 1397 1398
  if (!p1) {
    // window has been closed
    return false;
  }
5
54liuyao 已提交
1399
  // SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId);
1400
  // dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage);
1401
  tSimpleHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1402 1403 1404
  return true;
}

L
Liu Jicong 已提交
1405 1406
void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pDelWins, SInterval* pInterval,
                                   SHashObj* pUpdatedMap) {
1407
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
1408
  TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
1409 1410
  SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           tsEnds = (TSKEY*)pEndCol->pData;
1411
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
1412
  uint64_t*        groupIds = (uint64_t*)pGroupCol->pData;
1413 1414 1415
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
1416
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC);
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
    do {
      doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
      SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]};
      if (pDelWins) {
        taosArrayPush(pDelWins, &winRes);
      }
      if (pUpdatedMap) {
        taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
    } while (win.skey < tsEnds[i]);
1428 1429 1430
  }
}

1431 1432
static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput,
                           SSDataBlock* pBlock, SArray* pUpWins) {
1433 1434 1435 1436
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  TSKEY*           startTsCols = (TSKEY*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           endTsCols = (TSKEY*)pEndTsCol->pData;
1437 1438
  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*        pGpDatas = (uint64_t*)pGpCol->pData;
1439 1440 1441 1442 1443
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
    while (win.ekey <= endTsCols[i]) {
1444
      uint64_t winGpId = pGpDatas[i];
L
Liu Jicong 已提交
1445
      bool     res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput);
1446
      if (pUpWins && res) {
H
Haojun Liao 已提交
1447
        SWinKey winRes = {.ts = win.skey, .groupId = winGpId};
1448 1449 1450
        taosArrayPush(pUpWins, &winRes);
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
5
54liuyao 已提交
1451
    }
5
54liuyao 已提交
1452 1453
  }
}
1454

1455 1456 1457 1458 1459 1460
static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) {
  void*   pIte = NULL;
  size_t  keyLen = 0;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    void*    key = tSimpleHashGetKey(pIte, &keyLen);
5
54liuyao 已提交
1461 1462
    uint64_t groupId = *(uint64_t*)key;
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
1463
    TSKEY               ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1464
    SResultRowPosition* pPos = (SResultRowPosition*)pIte;
5
54liuyao 已提交
1465
    int32_t             code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
5
54liuyao 已提交
1466 1467 1468 1469 1470 1471 1472
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }
  return TSDB_CODE_SUCCESS;
}

1473
static int32_t closeIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
5
54liuyao 已提交
1474
                                   SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pRecyPages,
1475
                                   SDiskbasedBuf* pDiscBuf) {
1476
  qDebug("===stream===close interval window");
1477 1478 1479 1480 1481
  void*   pIte = NULL;
  size_t  keyLen = 0;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    void*    key = tSimpleHashGetKey(pIte, &keyLen);
X
Xiaoyu Wang 已提交
1482
    uint64_t groupId = *(uint64_t*)key;
5
54liuyao 已提交
1483
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
L
Liu Jicong 已提交
1484
    TSKEY       ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1485 1486 1487
    STimeWindow win;
    win.skey = ts;
    win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
H
Haojun Liao 已提交
1488
    SWinKey winRe = {
L
Liu Jicong 已提交
1489 1490
        .ts = win.skey,
        .groupId = groupId,
L
Liu Jicong 已提交
1491
    };
H
Haojun Liao 已提交
1492
    void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinKey));
5
54liuyao 已提交
1493
    if (isCloseWindow(&win, pSup)) {
5
54liuyao 已提交
1494
      if (chIds && pPullDataMap) {
L
Liu Jicong 已提交
1495
        SArray* chAy = *(SArray**)chIds;
5
54liuyao 已提交
1496
        int32_t size = taosArrayGetSize(chAy);
5
54liuyao 已提交
1497
        qDebug("===stream===window %" PRId64 " wait child size:%d", win.skey, size);
5
54liuyao 已提交
1498
        for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
1499
          qDebug("===stream===window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i));
5
54liuyao 已提交
1500 1501 1502
        }
        continue;
      } else if (pPullDataMap) {
5
54liuyao 已提交
1503
        qDebug("===stream===close window %" PRId64, win.skey);
5
54liuyao 已提交
1504
      }
5
54liuyao 已提交
1505 1506
      SResultRowPosition* pPos = (SResultRowPosition*)pIte;
      if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
1507
        int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, closeWins);
5
54liuyao 已提交
1508 1509 1510
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
1511 1512 1513
        ASSERT(pRecyPages != NULL);
        taosArrayPush(pRecyPages, &pPos->pageId);
      } else {
5
54liuyao 已提交
1514
        // SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId);
1515
        // dBufSetBufPageRecycled(pDiscBuf, bufPage);
5
54liuyao 已提交
1516
      }
5
54liuyao 已提交
1517 1518
      char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
      SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
1519
      tSimpleHashIterateRemove(pHashMap, keyBuf, keyLen, &pIte, &iter);
5
54liuyao 已提交
1520 1521 1522 1523 1524
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
1525 1526 1527
static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) {
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
L
Liu Jicong 已提交
1528
    SOperatorInfo*                    pChildOp = taosArrayGetP(pChildren, i);
5
54liuyao 已提交
1529 1530 1531
    SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
    ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
1532 1533
    closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, NULL,
                        pChInfo->aggSup.pResultBuf);
1534 1535 1536 1537 1538 1539 1540
  }
}

static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) {
  int32_t size = taosArrayGetSize(pageIds);
  for (int32_t i = 0; i < size; i++) {
    int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i);
5
54liuyao 已提交
1541
    // SFilePage* bufPage = getBufPage(pDiskBuf, pageId);
1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555
    // dBufSetBufPageRecycled(pDiskBuf, bufPage);
  }
  taosArrayClear(pageIds);
}

static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlock) {
  blockDataCleanup(pBlock);
  int32_t size = taosArrayGetSize(pWins);
  if (*index == size) {
    *index = 0;
    taosArrayClear(pWins);
    return;
  }
  blockDataEnsureCapacity(pBlock, size - *index);
1556
  uint64_t uid = 0;
1557
  for (int32_t i = *index; i < size; i++) {
H
Haojun Liao 已提交
1558
    SWinKey* pWin = taosArrayGet(pWins, i);
1559
    appendOneRow(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId);
1560
    (*index)++;
5
54liuyao 已提交
1561 1562 1563
  }
}

1564
static void destroyStateWindowOperatorInfo(void* param) {
1565
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
1566
  cleanupBasicInfo(&pInfo->binfo);
1567
  taosMemoryFreeClear(pInfo->stateKey.pData);
1568

D
dapan1121 已提交
1569
  taosMemoryFreeClear(param);
1570 1571
}

H
Haojun Liao 已提交
1572
static void freeItem(void* param) {
L
Liu Jicong 已提交
1573
  SGroupKeys* pKey = (SGroupKeys*)param;
H
Haojun Liao 已提交
1574 1575 1576
  taosMemoryFree(pKey->pData);
}

1577
void destroyIntervalOperatorInfo(void* param) {
1578
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
1579
  cleanupBasicInfo(&pInfo->binfo);
1580
  cleanupAggSup(&pInfo->aggSup);
1581 1582 1583 1584
  cleanupExprSupp(&pInfo->scalarSupp);

  tdListFree(pInfo->binfo.resultRowInfo.openWindow);

H
Haojun Liao 已提交
1585 1586 1587 1588 1589 1590 1591
  pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages);
  pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols);
  taosArrayDestroyEx(pInfo->pPrevValues, freeItem);

  pInfo->pPrevValues = NULL;
  pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins);
  pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);
1592

H
Haojun Liao 已提交
1593 1594
  cleanupGroupResInfo(&pInfo->groupResInfo);
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
D
dapan1121 已提交
1595
  taosMemoryFreeClear(param);
1596 1597
}

1598
void destroyStreamFinalIntervalOperatorInfo(void* param) {
X
Xiaoyu Wang 已提交
1599
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
1600
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
1601
  cleanupAggSup(&pInfo->aggSup);
L
Liu Jicong 已提交
1602
  // it should be empty.
5
54liuyao 已提交
1603 1604 1605
  taosHashCleanup(pInfo->pPullDataMap);
  taosArrayDestroy(pInfo->pPullWins);
  blockDataDestroy(pInfo->pPullDataRes);
1606
  taosArrayDestroy(pInfo->pRecycledPages);
H
Haojun Liao 已提交
1607
  blockDataDestroy(pInfo->pUpdateRes);
L
Liu Jicong 已提交
1608 1609
  taosArrayDestroy(pInfo->pDelWins);
  blockDataDestroy(pInfo->pDelRes);
5
54liuyao 已提交
1610

1611 1612 1613 1614
  if (pInfo->pChildren) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
1615
      destroyStreamFinalIntervalOperatorInfo(pChildOp->info);
L
Liu Jicong 已提交
1616 1617
      taosMemoryFree(pChildOp->pDownstream);
      cleanupExprSupp(&pChildOp->exprSupp);
1618 1619
      taosMemoryFreeClear(pChildOp);
    }
L
Liu Jicong 已提交
1620
    taosArrayDestroy(pInfo->pChildren);
1621
  }
1622
  nodesDestroyNode((SNode*)pInfo->pPhyNode);
5
54liuyao 已提交
1623
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
1624

D
dapan1121 已提交
1625
  taosMemoryFreeClear(param);
5
54liuyao 已提交
1626 1627
}

1628
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
5
54liuyao 已提交
1629
  for (int32_t i = 0; i < numOfCols; i++) {
5
54liuyao 已提交
1630
    if (fmIsUserDefinedFunc(pFCtx[i].functionId) || !fmIsInvertible(pFCtx[i].functionId)) {
5
54liuyao 已提交
1631 1632 1633 1634 1635 1636
      return false;
    }
  }
  return true;
}

1637
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
1638 1639
  // the primary timestamp column
  bool needed = false;
1640 1641
  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
1642

X
Xiaoyu Wang 已提交
1643
  {  // ts column
1644 1645
    SColumn c = {0};
    c.colId = 1;
1646
    c.slotId = pInfo->primaryTsIndex;
1647 1648
    c.type = TSDB_DATA_TYPE_TIMESTAMP;
    c.bytes = sizeof(int64_t);
1649
    taosArrayPush(pInfo->pInterpCols, &c);
1650 1651

    SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1652 1653 1654 1655
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = true;  // to denote no value is assigned yet
    key.pData = taosMemoryCalloc(1, c.bytes);
1656
    taosArrayPush(pInfo->pPrevValues, &key);
1657 1658
  }

X
Xiaoyu Wang 已提交
1659
  for (int32_t i = 0; i < numOfCols; ++i) {
1660 1661
    SExprInfo* pExpr = pCtx[i].pExpr;

H
Haojun Liao 已提交
1662
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1663 1664 1665
      SFunctParam* pParam = &pExpr->base.pParam[0];

      SColumn c = *pParam->pCol;
1666
      taosArrayPush(pInfo->pInterpCols, &c);
1667 1668 1669
      needed = true;

      SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1670 1671
      key.bytes = c.bytes;
      key.type = c.type;
1672
      key.isNull = false;
X
Xiaoyu Wang 已提交
1673
      key.pData = taosMemoryCalloc(1, c.bytes);
1674
      taosArrayPush(pInfo->pPrevValues, &key);
1675 1676 1677 1678 1679 1680
    }
  }

  return needed;
}

L
Liu Jicong 已提交
1681 1682
void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval,
                            int64_t waterMark) {
1683
  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
1684
    initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, waterMark);
1685 1686
    return;
  }
5
54liuyao 已提交
1687
  SStreamScanInfo* pScanInfo = downstream->info;
1688 1689 1690 1691
  pScanInfo->windowSup.parentType = type;
  pScanInfo->windowSup.pIntervalAggSup = pSup;
  pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, waterMark);
  pScanInfo->interval = *pInterval;
5
54liuyao 已提交
1692 1693
}

H
Haojun Liao 已提交
1694 1695 1696 1697 1698 1699
void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
  for (int32_t i = 0; i < numOfExpr; i++) {
    pCtx[i].isStream = true;
  }
}

1700 1701
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                          SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
L
Liu Jicong 已提交
1702 1703
                                          STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode,
                                          SExecTaskInfo* pTaskInfo, bool isStream) {
1704
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1705
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1706 1707 1708 1709
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1710
  pOperator->pTaskInfo = pTaskInfo;
L
Liu Jicong 已提交
1711
  pInfo->win = pTaskInfo->window;
1712 1713
  pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
  pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
L
Liu Jicong 已提交
1714
  pInfo->interval = *pInterval;
L
Liu Jicong 已提交
1715
  pInfo->execModel = pTaskInfo->execModel;
L
Liu Jicong 已提交
1716
  pInfo->twAggSup = *pTwAggSupp;
5
54liuyao 已提交
1717
  pInfo->ignoreExpiredData = pPhyNode->window.igExpired;
1718
  pInfo->pCondition = pPhyNode->window.node.pConditions;
1719
  pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock;
1720 1721 1722 1723

  if (pPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
L
Liu Jicong 已提交
1724
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
1725 1726 1727 1728
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
1729

1730
  pInfo->primaryTsIndex = primaryTsSlotId;
1731 1732
  SExprSupp* pSup = &pOperator->exprSupp;

1733
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
1734
  initResultSizeInfo(&pOperator->resultInfo, 4096);
1735

1736
  int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
1737 1738 1739 1740
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

1741 1742
  initBasicInfo(&pInfo->binfo, pResBlock);

1743 1744
  if (isStream) {
    ASSERT(numOfCols > 0);
H
Haojun Liao 已提交
1745
    initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
1746
  }
1747

1748
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
1749

1750
  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
X
Xiaoyu Wang 已提交
1751
  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
1752

1753
  pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, pInfo);
1754
  if (pInfo->timeWindowInterpo) {
1755
    pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
H
Haojun Liao 已提交
1756 1757 1758
    if (pInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
1759
  }
1760

1761
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
H
Haojun Liao 已提交
1762
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
1763
  pInfo->delIndex = 0;
1764
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
1765
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1766

X
Xiaoyu Wang 已提交
1767 1768 1769 1770 1771
  pOperator->name = "TimeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
1772

1773
  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL,
1774 1775
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

5
54liuyao 已提交
1776
  if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) {
L
Liu Jicong 已提交
1777 1778
    initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup, &pInfo->interval,
                           pInfo->twAggSup.waterMark);
5
54liuyao 已提交
1779 1780
  }

1781 1782 1783 1784 1785 1786 1787
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1788
_error:
1789
  destroyIntervalOperatorInfo(pInfo);
1790 1791 1792 1793 1794
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

1795
// todo handle multiple timeline cases. assume no timeline interweaving
1796 1797
static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1798
  SExprSupp*     pSup = &pOperator->exprSupp;
1799

1800
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1801 1802

  bool    masterScan = true;
1803
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818
  int64_t gid = pBlock->info.groupId;

  int64_t gap = pInfo->gap;

  if (!pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->winSup.prevTs = INT64_MIN;
  }

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

  // In case of ascending or descending order scan data, only one time window needs to be kepted for each table.
  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
1819 1820 1821
    if (gid != pRowSup->groupId || pInfo->winSup.prevTs == INT64_MIN) {
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
H
Haojun Liao 已提交
1822 1823
    } else if (((tsList[j] - pRowSup->prevTs >= 0) && (tsList[j] - pRowSup->prevTs <= gap)) ||
               ((pRowSup->prevTs - tsList[j] >= 0) && (pRowSup->prevTs - tsList[j] <= gap))) {
1824
      // The gap is less than the threshold, so it belongs to current session window that has been opened already.
1825
      doKeepTuple(pRowSup, tsList[j], gid);
1826 1827 1828 1829 1830 1831 1832 1833 1834 1835
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1836 1837
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1838
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1839
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1840 1841 1842 1843
      }

      // pInfo->numOfRows data belong to the current session window
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1844 1845
      doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                       pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1846 1847

      // here we start a new session window
1848 1849
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1850 1851 1852 1853 1854
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1855 1856
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1857
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1858
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1859 1860 1861
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
H
Haojun Liao 已提交
1862 1863
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
                   pBlock->info.rows, numOfOutput);
1864 1865
}

1866
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
1867 1868 1869 1870 1871 1872
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*          pBInfo = &pInfo->binfo;
1873
  SExprSupp*               pSup = &pOperator->exprSupp;
1874 1875

  if (pOperator->status == OP_RES_TO_RETURN) {
1876
    while (1) {
1877
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1878
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1879

1880
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1881 1882 1883 1884
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
1885

1886 1887 1888 1889 1890
      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1891
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1892 1893
  }

1894 1895 1896
  int64_t st = taosGetTimestampUs();
  int32_t order = TSDB_ORDER_ASC;

1897 1898 1899
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1900
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1901 1902 1903 1904 1905
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1906
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1907 1908
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1909 1910 1911
    doSessionWindowAggImpl(pOperator, pInfo, pBlock);
  }

1912 1913
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;

1914 1915 1916
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;

1917
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1918
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1919
  while (1) {
1920
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1921
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1922

1923
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1924 1925 1926 1927
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
1928

1929 1930 1931 1932 1933
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1934
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1935 1936
}

1937
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
H
Haojun Liao 已提交
1938
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
1939 1940
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
H
Haojun Liao 已提交
1941 1942 1943 1944 1945 1946

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);

      pkey->isNull = false;
1947
      char* val = colDataGetData(pColInfoData, rowIndex);
H
Haojun Liao 已提交
1948 1949 1950
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }
1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970

  pSliceInfo->isPrevRowSet = true;
}

static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i);

      pkey->isNull = false;
      char* val = colDataGetData(pColInfoData, rowIndex);
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }

  pSliceInfo->isNextRowSet = true;
H
Haojun Liao 已提交
1971 1972
}

1973 1974
static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex,
                             bool isLastRow) {
1975
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
dengyihao's avatar
dengyihao 已提交
1976
  bool    fillLastPoint = pSliceInfo->fillLastPoint;
1977 1978
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
1979 1980
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
    SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i);
1981 1982 1983

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
      if (isLastRow) {
        pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
      } else if (fillLastPoint) {
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
      } else {
        pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1);

        char* val;
        val = colDataGetData(pColInfoData, rowIndex);
        memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes);
        val = colDataGetData(pColInfoData, rowIndex + 1);
        memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes);
      }
2000 2001 2002 2003

      pLinearInfo->hasNull = false;
    } else {
      pLinearInfo->hasNull = true;
2004 2005 2006
    }
  }

2007
  pSliceInfo->fillLastPoint = isLastRow ? true : false;
2008 2009
}

dengyihao's avatar
dengyihao 已提交
2010
static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) {
2011
  int32_t rows = pResBlock->info.rows;
2012
  blockDataEnsureCapacity(pResBlock, rows + 1);
2013 2014 2015
  // todo set the correct primary timestamp column

  // output the result
2016
  bool hasInterp = true;
2017 2018 2019
  for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
    SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
    int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
2020
    int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
2021

dengyihao's avatar
dengyihao 已提交
2022
    // SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
2023 2024 2025
    SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

    switch (pSliceInfo->fillType) {
2026
      case TSDB_FILL_NULL: {
2027 2028
        colDataAppendNULL(pDst, rows);
        break;
2029
      }
2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046

      case TSDB_FILL_SET_VALUE: {
        SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal;

        if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
          float v = 0;
          GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
          double v = 0;
          GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
          int64_t v = 0;
          GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        }
2047 2048
        break;
      }
2049

2050
      case TSDB_FILL_LINEAR: {
2051
        SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, srcSlot);
2052

dengyihao's avatar
dengyihao 已提交
2053 2054
        SPoint start = pLinearInfo->start;
        SPoint end = pLinearInfo->end;
2055 2056 2057
        SPoint current = {.key = pSliceInfo->current};
        current.val = taosMemoryCalloc(pLinearInfo->bytes, 1);

2058 2059
        // before interp range, do not fill
        if (start.key == INT64_MIN || end.key == INT64_MAX) {
2060
          hasInterp = false;
2061 2062 2063
          break;
        }

2064 2065 2066 2067
        if (pLinearInfo->hasNull) {
          colDataAppendNULL(pDst, rows);
        } else {
          taosGetLinearInterpolationVal(&current, pLinearInfo->type, &start, &end, pLinearInfo->type);
dengyihao's avatar
dengyihao 已提交
2068
          colDataAppend(pDst, rows, (char*)current.val, false);
2069 2070
        }

2071
        taosMemoryFree(current.val);
2072
        break;
2073
      }
2074
      case TSDB_FILL_PREV: {
2075
        if (!pSliceInfo->isPrevRowSet) {
2076
          hasInterp = false;
2077 2078 2079
          break;
        }

2080 2081
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
        colDataAppend(pDst, rows, pkey->pData, false);
2082 2083
        break;
      }
2084 2085

      case TSDB_FILL_NEXT: {
2086
        if (!pSliceInfo->isNextRowSet) {
2087
          hasInterp = false;
2088 2089 2090
          break;
        }

2091 2092
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot);
        colDataAppend(pDst, rows, pkey->pData, false);
2093 2094
        break;
      }
2095 2096 2097 2098 2099 2100

      case TSDB_FILL_NONE:
      default:
        break;
    }
  }
2101 2102 2103 2104

  if (hasInterp) {
    pResBlock->info.rows += 1;
  }
2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116
}

static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pPrevRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pPrevRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

2117
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pPrevRow, &key);
  }

2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157
  pInfo->isPrevRowSet = false;

  return TSDB_CODE_SUCCESS;
}

static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pNextRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pNextRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pNextRow, &key);
  }

  pInfo->isNextRowSet = false;

2158 2159 2160
  return TSDB_CODE_SUCCESS;
}

2161 2162 2163 2164 2165 2166
static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pLinearInfo != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo));
2167
  if (pInfo->pLinearInfo == NULL) {
2168 2169 2170 2171 2172 2173 2174
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

2175 2176
    SFillLinearInfo linearInfo = {0};
    linearInfo.start.key = INT64_MIN;
dengyihao's avatar
dengyihao 已提交
2177
    linearInfo.end.key = INT64_MAX;
2178
    linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes);
dengyihao's avatar
dengyihao 已提交
2179 2180 2181
    linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes);
    linearInfo.hasNull = false;
    linearInfo.type = pColInfo->info.type;
2182 2183
    linearInfo.bytes = pColInfo->info.bytes;
    taosArrayPush(pInfo->pLinearInfo, &linearInfo);
2184 2185
  }

2186 2187
  pInfo->fillLastPoint = false;

2188 2189 2190
  return TSDB_CODE_SUCCESS;
}

2191 2192 2193 2194
static bool needToFillLastPoint(STimeSliceOperatorInfo* pSliceInfo) {
  return (pSliceInfo->fillLastPoint == true && pSliceInfo->fillType == TSDB_FILL_LINEAR);
}

2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211
static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  int32_t code;
  code = initPrevRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initNextRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initFillLinearInfo(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

2212 2213 2214
  return TSDB_CODE_SUCCESS;
}

H
Haojun Liao 已提交
2215
static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
2216 2217 2218 2219
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

2220 2221
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

2222
  STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
2223 2224
  SSDataBlock*            pResBlock = pSliceInfo->pRes;
  SExprSupp*              pSup = &pOperator->exprSupp;
H
Haojun Liao 已提交
2225

2226 2227
  //  if (pOperator->status == OP_RES_TO_RETURN) {
  //    //    doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes);
2228
  //    if (pResBlock->info.rows == 0 || !hasRemainResults(&pSliceInfo->groupResInfo)) {
2229 2230 2231 2232 2233
  //      doSetOperatorCompleted(pOperator);
  //    }
  //
  //    return pResBlock;
  //  }
2234

2235 2236
  int32_t        order = TSDB_ORDER_ASC;
  SInterval*     pInterval = &pSliceInfo->interval;
2237 2238
  SOperatorInfo* downstream = pOperator->pDownstream[0];

2239 2240
  blockDataCleanup(pResBlock);

2241
  while (1) {
2242
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
2243 2244 2245 2246
    if (pBlock == NULL) {
      break;
    }

2247
    int32_t code = initKeeperInfo(pSliceInfo, pBlock);
2248
    if (code != TSDB_CODE_SUCCESS) {
2249
      T_LONG_JMP(pTaskInfo->env, code);
2250 2251
    }

2252
    // the pDataBlock are always the same one, no need to call this again
2253
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
H
Haojun Liao 已提交
2254

2255
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
2256 2257
    for (int32_t i = 0; i < pBlock->info.rows; ++i) {
      int64_t ts = *(int64_t*)colDataGetData(pTsCol, i);
H
Haojun Liao 已提交
2258

dengyihao's avatar
dengyihao 已提交
2259
      if (i == 0 && needToFillLastPoint(pSliceInfo)) {  // first row in current block
2260 2261 2262 2263 2264 2265 2266 2267 2268
        doKeepLinearInfo(pSliceInfo, pBlock, i, false);
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
          genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
        }
2269
      }
2270

2271 2272 2273
      if (pSliceInfo->current > pSliceInfo->win.ekey) {
        doSetOperatorCompleted(pOperator);
        break;
2274 2275
      }

H
Haojun Liao 已提交
2276
      if (ts == pSliceInfo->current) {
2277
        for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
2278
          SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
2279 2280
          int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
          int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
H
Haojun Liao 已提交
2281 2282

          SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
2283
          SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
H
Haojun Liao 已提交
2284

2285 2286 2287 2288 2289
          if (colDataIsNull_s(pSrc, i)) {
            colDataAppendNULL(pDst, pResBlock->info.rows);
            continue;
          }

H
Haojun Liao 已提交
2290
          char* v = colDataGetData(pSrc, i);
2291
          colDataAppend(pDst, pResBlock->info.rows, v, false);
H
Haojun Liao 已提交
2292 2293
        }

2294
        pResBlock->info.rows += 1;
2295
        doKeepPrevRows(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2296

2297 2298
        // for linear interpolation, always fill value between this and next points;
        // if its the first point in data block, also fill values between previous(if there's any) and this point;
2299 2300
        // if its the last point in data block, no need to fill, but reserve this point as the start value and do
        // the interpolation when processing next data block.
2301
        if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2302 2303
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2304
          if (i < pBlock->info.rows - 1) {
2305
            doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2306 2307 2308
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2309
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2310 2311
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2312 2313 2314 2315
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }
2316

2317 2318 2319 2320 2321
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
            }
dengyihao's avatar
dengyihao 已提交
2322 2323
          } else {  // it is the last row of current block
            // store ts value as start, and calculate interp value when processing next block
2324
            doKeepLinearInfo(pSliceInfo, pBlock, i, true);
2325
          }
dengyihao's avatar
dengyihao 已提交
2326
        } else {  // non-linear interpolation
2327 2328 2329 2330 2331 2332
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (pSliceInfo->current > pSliceInfo->win.ekey) {
            doSetOperatorCompleted(pOperator);
            break;
          }
2333

2334 2335 2336
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
2337
        }
H
Haojun Liao 已提交
2338
      } else if (ts < pSliceInfo->current) {
2339
        // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate
2340 2341
        doKeepPrevRows(pSliceInfo, pBlock, i);

2342
        if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2343
          // no need to increate pSliceInfo->current here
dengyihao's avatar
dengyihao 已提交
2344
          // pSliceInfo->current =
2345 2346
          //    taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (i < pBlock->info.rows - 1) {
2347
            doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2348 2349 2350
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2351
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2352 2353
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2354 2355 2356 2357 2358 2359 2360
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }

              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
H
Haojun Liao 已提交
2361 2362
                break;
              }
H
Haojun Liao 已提交
2363
            }
2364
          }
dengyihao's avatar
dengyihao 已提交
2365
        } else {  // non-linear interpolation
2366 2367 2368 2369 2370 2371
          if (i < pBlock->info.rows - 1) {
            // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
            doKeepNextRows(pSliceInfo, pBlock, i + 1);
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2372
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2373 2374
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2375 2376 2377 2378
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }
2379

2380 2381 2382 2383 2384 2385
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
            } else {
              // ignore current row, and do nothing
H
Haojun Liao 已提交
2386
            }
2387 2388
          } else {  // it is the last row of current block
            doKeepPrevRows(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2389
          }
2390 2391
        }
      } else {  // ts > pSliceInfo->current
2392
        // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
2393 2394
        doKeepNextRows(pSliceInfo, pBlock, i);

2395
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
2396
          genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
2397 2398
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2399 2400 2401
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
2402 2403
        }

2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420
        // add current row if timestamp match
        if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
          for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
            SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
            int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
            int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;

            SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
            SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

            char* v = colDataGetData(pSrc, i);
            colDataAppend(pDst, pResBlock->info.rows, v, false);
          }

          pResBlock->info.rows += 1;
          doKeepPrevRows(pSliceInfo, pBlock, i);

2421 2422 2423 2424
          if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
            pSliceInfo->current =
                taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
            if (i < pBlock->info.rows - 1) {
2425
              doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2426 2427 2428
              int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
              if (nextTs > pSliceInfo->current) {
                while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2429
                  genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2430 2431
                  pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                    pInterval->precision);
2432 2433 2434 2435
                  if (pResBlock->info.rows >= pResBlock->info.capacity) {
                    break;
                  }
                }
2436

2437 2438 2439 2440 2441 2442
                if (pSliceInfo->current > pSliceInfo->win.ekey) {
                  doSetOperatorCompleted(pOperator);
                  break;
                }
              }
            }
dengyihao's avatar
dengyihao 已提交
2443
          } else {  // non-linear interpolation
2444 2445 2446 2447 2448 2449
            pSliceInfo->current =
                taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);

            if (pResBlock->info.rows >= pResBlock->info.capacity) {
              break;
            }
2450 2451 2452
          }
        }

2453 2454 2455
        if (pSliceInfo->current > pSliceInfo->win.ekey) {
          doSetOperatorCompleted(pOperator);
          break;
H
Haojun Liao 已提交
2456 2457 2458
        }
      }
    }
2459
  }
2460

2461 2462
  // check if need to interpolate after last datablock
  // except for fill(next), fill(linear)
dengyihao's avatar
dengyihao 已提交
2463 2464
  while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT &&
         pSliceInfo->fillType != TSDB_FILL_LINEAR) {
2465 2466 2467 2468 2469
    genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
    pSliceInfo->current =
        taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
    if (pResBlock->info.rows >= pResBlock->info.capacity) {
      break;
2470
    }
2471 2472 2473 2474
  }

  // restore the value
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
2475
  if (pResBlock->info.rows == 0) {
2476 2477 2478
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
2479 2480 2481
  return pResBlock->info.rows == 0 ? NULL : pResBlock;
}

2482
void destroyTimeSliceOperatorInfo(void* param) {
2483 2484 2485 2486 2487 2488 2489 2490
  STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param;

  pInfo->pRes = blockDataDestroy(pInfo->pRes);

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) {
    SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i);
    taosMemoryFree(pKey->pData);
  }
2491
  taosArrayDestroy(pInfo->pPrevRow);
2492 2493 2494 2495 2496 2497

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pNextRow); ++i) {
    SGroupKeys* pKey = taosArrayGet(pInfo->pNextRow, i);
    taosMemoryFree(pKey->pData);
  }
  taosArrayDestroy(pInfo->pNextRow);
2498 2499 2500 2501 2502 2503

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pLinearInfo); ++i) {
    SFillLinearInfo* pKey = taosArrayGet(pInfo->pLinearInfo, i);
    taosMemoryFree(pKey->start.val);
    taosMemoryFree(pKey->end.val);
  }
2504 2505
  taosArrayDestroy(pInfo->pLinearInfo);

2506
  taosMemoryFree(pInfo->pFillColInfo);
2507 2508 2509
  taosMemoryFreeClear(param);
}

2510
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
2511 2512 2513 2514 2515 2516
  STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pOperator == NULL || pInfo == NULL) {
    goto _error;
  }

2517
  SInterpFuncPhysiNode* pInterpPhyNode = (SInterpFuncPhysiNode*)pPhyNode;
2518
  SExprSupp*            pSup = &pOperator->exprSupp;
2519

2520
  int32_t    numOfExprs = 0;
2521
  SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
2522
  int32_t    code = initExprSupp(pSup, pExprInfo, numOfExprs);
H
Haojun Liao 已提交
2523 2524 2525 2526
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2527
  if (pInterpPhyNode->pExprs != NULL) {
2528
    int32_t    num = 0;
2529 2530 2531 2532 2533 2534 2535 2536 2537
    SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

  pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries);
  pInfo->fillType = convertFillType(pInterpPhyNode->fillMode);
2538
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2539

H
Haojun Liao 已提交
2540
  pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, NULL, 0, (SNodeListNode*)pInterpPhyNode->pFillValues);
2541
  pInfo->pLinearInfo = NULL;
L
Liu Jicong 已提交
2542 2543
  pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  pInfo->win = pInterpPhyNode->timeRange;
2544
  pInfo->interval.interval = pInterpPhyNode->interval;
L
Liu Jicong 已提交
2545
  pInfo->current = pInfo->win.skey;
H
Haojun Liao 已提交
2546

2547
  pOperator->name = "TimeSliceOperator";
2548
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC;
2549 2550 2551 2552
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
  pOperator->pTaskInfo = pTaskInfo;
2553

2554
  pOperator->fpSet =
2555
      createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyTimeSliceOperatorInfo, NULL, NULL, NULL);
2556

2557 2558
  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);

H
Haojun Liao 已提交
2559
  code = appendDownstream(pOperator, &downstream, 1);
2560 2561
  return pOperator;

L
Liu Jicong 已提交
2562
_error:
2563 2564 2565 2566 2567 2568
  taosMemoryFree(pInfo);
  taosMemoryFree(pOperator);
  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
  return NULL;
}

2569 2570
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
                                             SExecTaskInfo* pTaskInfo) {
2571 2572 2573 2574 2575 2576
  SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

L
Liu Jicong 已提交
2577
  int32_t      num = 0;
2578 2579 2580 2581 2582 2583 2584
  SExprInfo*   pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc);
  int32_t      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;

  SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
2585 2586 2587
  pInfo->stateKey.type = pInfo->stateCol.type;
  pInfo->stateKey.bytes = pInfo->stateCol.bytes;
  pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
2588
  pInfo->pCondition = pStateNode->window.node.pConditions;
2589 2590 2591 2592
  if (pInfo->stateKey.pData == NULL) {
    goto _error;
  }

2593 2594
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

2595
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2596
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
2597 2598 2599 2600
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2601
  initBasicInfo(&pInfo->binfo, pResBlock);
2602
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2603

L
Liu Jicong 已提交
2604 2605 2606
  pInfo->twAggSup =
      (STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType};
  ;
2607 2608
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

X
Xiaoyu Wang 已提交
2609 2610
  pInfo->tsSlotId = tsSlotId;
  pOperator->name = "StateWindowOperator";
2611
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE;
X
Xiaoyu Wang 已提交
2612 2613 2614 2615
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
2616 2617 2618 2619

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL,
                                         destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

2620 2621 2622 2623 2624
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2625 2626
  return pOperator;

L
Liu Jicong 已提交
2627
_error:
2628 2629 2630
  destroyStateWindowOperatorInfo(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
2631 2632 2633
  return NULL;
}

2634
void destroySWindowOperatorInfo(void* param) {
2635
  SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
2636 2637 2638
  if (pInfo == NULL) {
    return;
  }
2639

2640
  cleanupBasicInfo(&pInfo->binfo);
H
Haojun Liao 已提交
2641 2642 2643 2644
  colDataDestroy(&pInfo->twAggSup.timeWindowData);

  cleanupAggSup(&pInfo->aggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
D
dapan1121 已提交
2645
  taosMemoryFreeClear(param);
2646 2647
}

H
Haojun Liao 已提交
2648
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode,
2649
                                            SExecTaskInfo* pTaskInfo) {
2650 2651 2652 2653 2654 2655
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

2656
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
2657
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2658

2659
  int32_t      numOfCols = 0;
H
Haojun Liao 已提交
2660 2661 2662
  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc);

2663
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
2664 2665 2666 2667
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2668 2669
  initBasicInfo(&pInfo->binfo, pResBlock);

H
Haojun Liao 已提交
2670 2671 2672 2673
  pInfo->twAggSup.waterMark = pSessionNode->window.watermark;
  pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType;
  pInfo->gap = pSessionNode->gap;

2674
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2675 2676
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

2677
  pInfo->tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
L
Liu Jicong 已提交
2678 2679 2680
  pInfo->binfo.pRes = pResBlock;
  pInfo->winSup.prevTs = INT64_MIN;
  pInfo->reptScan = false;
2681
  pInfo->pCondition = pSessionNode->window.node.pConditions;
H
Haojun Liao 已提交
2682

L
Liu Jicong 已提交
2683
  pOperator->name = "SessionWindowAggOperator";
2684
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
2685
  pOperator->blocking = true;
L
Liu Jicong 已提交
2686 2687
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
2688 2689 2690 2691 2692

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL,
                                         destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  pOperator->pTaskInfo = pTaskInfo;
  code = appendDownstream(pOperator, &downstream, 1);
2693 2694 2695 2696
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2697 2698
  return pOperator;

L
Liu Jicong 已提交
2699
_error:
2700
  destroySWindowOperatorInfo(pInfo);
2701 2702 2703
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
2704
}
5
54liuyao 已提交
2705

5
54liuyao 已提交
2706
void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
2707
                      SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) {
5
54liuyao 已提交
2708 2709
  for (int32_t k = 0; k < numOfOutput; ++k) {
    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
2710 2711 2712 2713 2714
      if (!pTimeWindowData) {
        continue;
      }

      SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);
L
Liu Jicong 已提交
2715 2716
      char*                p = GET_ROWCELL_INTERBUF(pEntryInfo);
      SColumnInfoData      idata = {0};
2717 2718 2719 2720 2721 2722 2723 2724
      idata.info.type = TSDB_DATA_TYPE_BIGINT;
      idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
      idata.pData = p;

      SScalarParam out = {.columnData = &idata};
      SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
      pDestCtx[k].sfp.process(&tw, 1, &out);
      pEntryInfo->numOfRes = 1;
L
Liu Jicong 已提交
2725
    } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
2726
      int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
5
54liuyao 已提交
2727 2728 2729
      if (code != TSDB_CODE_SUCCESS) {
        qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
        pTaskInfo->code = code;
2730
        T_LONG_JMP(pTaskInfo->env, code);
5
54liuyao 已提交
2731 2732 2733 2734 2735
      }
    }
  }
}

2736 2737 2738 2739
bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) {
  int32_t bytes = sizeof(TSKEY);
  SET_RES_WINDOW_KEY(pSup->keyBuf, &ts, bytes, groupId);
  SResultRowPosition* p1 =
2740
      (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
2741 2742 2743
  return p1 != NULL;
}

2744 2745 2746 2747 2748 2749
STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
  return w;
}

2750
static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray,
L
Liu Jicong 已提交
2751 2752
                                  int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo,
                                  SHashObj* pUpdatedMap) {
5
54liuyao 已提交
2753
  int32_t size = taosArrayGetSize(pWinArray);
5
54liuyao 已提交
2754 2755 2756
  if (!pInfo->pChildren) {
    return;
  }
5
54liuyao 已提交
2757
  for (int32_t i = 0; i < size; i++) {
H
Haojun Liao 已提交
2758
    SWinKey*    pWinRes = taosArrayGet(pWinArray, i);
2759
    SResultRow* pCurResult = NULL;
2760 2761 2762 2763 2764
    STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval);
    if (isDeletedWindow(&parentWin, pWinRes->groupId, &pInfo->aggSup) && isCloseWindow(&parentWin, &pInfo->twAggSup)) {
      continue;
    }
    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &parentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx,
2765
                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2766
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
2767
    int32_t num = 0;
5
54liuyao 已提交
2768 2769 2770
    for (int32_t j = 0; j < numOfChildren; j++) {
      SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
2771
      SExprSupp*                pChildSup = &pChildOp->exprSupp;
2772 2773 2774
      if (!hasIntervalWindow(&pChInfo->aggSup, pWinRes->ts, pWinRes->groupId)) {
        continue;
      }
2775
      num++;
2776
      SResultRow* pChResult = NULL;
2777
      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &parentWin, true, &pChResult, pWinRes->groupId,
2778 2779
                             pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup,
                             pTaskInfo);
2780 2781
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true);
      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
2782
    }
2783
    if (num > 0 && pUpdatedMap) {
2784
      saveWinResultRow(pCurResult, pWinRes->groupId, pUpdatedMap);
D
dapan1121 已提交
2785
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur);
2786
    }
5
54liuyao 已提交
2787 2788 2789 2790 2791
  }
}

bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
  SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId);
2792
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
L
Liu Jicong 已提交
2793
                                                               GET_RES_WINDOW_KEY_LEN(sizeof(int64_t)));
5
54liuyao 已提交
2794 2795 2796
  return p1 == NULL;
}

L
Liu Jicong 已提交
2797 2798 2799 2800
int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,
                        STimeWindow* pNextWin) {
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2801 2802 2803 2804
  int32_t prevEndPos = forwardRows - 1 + startPos;
  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
}

H
Haojun Liao 已提交
2805
void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {
5
54liuyao 已提交
2806 2807 2808 2809
  SArray* childIds = taosArrayInit(8, sizeof(int32_t));
  for (int32_t i = 0; i < size; i++) {
    taosArrayPush(childIds, &i);
  }
H
Haojun Liao 已提交
2810
  taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*));
5
54liuyao 已提交
2811 2812 2813 2814
}

static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

2815 2816
static void doHashIntervalAgg(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
                              SHashObj* pUpdatedMap) {
5
54liuyao 已提交
2817
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
X
Xiaoyu Wang 已提交
2818 2819
  SResultRowInfo*                   pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*                    pTaskInfo = pOperatorInfo->pTaskInfo;
2820 2821
  SExprSupp*                        pSup = &pOperatorInfo->exprSupp;
  int32_t                           numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
2822 2823 2824 2825
  int32_t                           step = 1;
  TSKEY*                            tsCols = NULL;
  SResultRow*                       pResult = NULL;
  int32_t                           forwardRows = 0;
5
54liuyao 已提交
2826

5
54liuyao 已提交
2827 2828 2829
  ASSERT(pSDataBlock->pDataBlock != NULL);
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
2830

2831
  int32_t     startPos = 0;
X
Xiaoyu Wang 已提交
2832
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
5
54liuyao 已提交
2833 2834 2835 2836 2837 2838
  STimeWindow nextWin = {0};
  if (IS_FINAL_OP(pInfo)) {
    nextWin = getFinalTimeWindow(ts, &pInfo->interval);
  } else {
    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order);
  }
5
54liuyao 已提交
2839
  while (1) {
5
54liuyao 已提交
2840
    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
5
54liuyao 已提交
2841
    if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
5
54liuyao 已提交
2842 2843 2844 2845 2846 2847 2848
      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
      if (startPos < 0) {
        break;
      }
      continue;
    }
    if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) {
L
Liu Jicong 已提交
2849
      bool    ignore = true;
H
Haojun Liao 已提交
2850
      SWinKey winRes = {
L
Liu Jicong 已提交
2851 2852 2853
          .ts = nextWin.skey,
          .groupId = tableGroupId,
      };
H
Haojun Liao 已提交
2854
      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
2855 2856 2857
      if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId};
        // add pull data request
5
54liuyao 已提交
2858
        savePullWindow(&pull, pInfo->pPullWins);
5
54liuyao 已提交
2859 2860 2861
        int32_t size = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, &winRes, size);
        qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size);
5
54liuyao 已提交
2862 2863 2864
      } else {
        int32_t index = -1;
        SArray* chArray = NULL;
5
54liuyao 已提交
2865
        int32_t chId = 0;
5
54liuyao 已提交
2866
        if (chIds) {
L
Liu Jicong 已提交
2867
          chArray = *(void**)chIds;
5
54liuyao 已提交
2868
          chId = getChildIndex(pSDataBlock);
5
54liuyao 已提交
2869 2870
          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
        }
L
Liu Jicong 已提交
2871
        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {
5
54liuyao 已提交
2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882
          ignore = false;
        }
      }

      if (ignore) {
        startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
        if (startPos < 0) {
          break;
        }
        continue;
      }
5
54liuyao 已提交
2883
    }
5
54liuyao 已提交
2884

2885 2886
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx,
                                          numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2887
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
2888
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
2889
    }
5
54liuyao 已提交
2890

5
54liuyao 已提交
2891 2892 2893
    if (IS_FINAL_OP(pInfo)) {
      forwardRows = 1;
    } else {
H
Hongze Cheng 已提交
2894 2895
      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,
                                             NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2896
    }
5
54liuyao 已提交
2897 2898
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
D
dapan1121 已提交
2899
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
5
54liuyao 已提交
2900
    }
5
54liuyao 已提交
2901
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
2902 2903
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pSDataBlock->info.rows, numOfOutput);
2904
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
2905
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
5
54liuyao 已提交
2906 2907 2908 2909 2910 2911 2912
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
    if (startPos < 0) {
      break;
    }
  }
}

5
54liuyao 已提交
2913
static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) {
2914
  tSimpleHashClear(pInfo->aggSup.pResultRowHashTable);
5
54liuyao 已提交
2915
  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
2916
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
L
Liu Jicong 已提交
2917
  pInfo->aggSup.currentPageId = -1;
5
54liuyao 已提交
2918 2919
}

5
54liuyao 已提交
2920 2921 2922 2923
static void clearSpecialDataBlock(SSDataBlock* pBlock) {
  if (pBlock->info.rows <= 0) {
    return;
  }
5
54liuyao 已提交
2924 2925 2926
  blockDataCleanup(pBlock);
}

2927
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
5
54liuyao 已提交
2928 2929
  // ASSERT(pDest->info.capacity >= pSource->info.rows);
  blockDataEnsureCapacity(pDest, pSource->info.rows);
5
54liuyao 已提交
2930
  clearSpecialDataBlock(pDest);
5
54liuyao 已提交
2931 2932
  SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
  SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
2933

5
54liuyao 已提交
2934
  // copy timestamp column
2935 2936
  colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info);
  for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) {
5
54liuyao 已提交
2937 2938 2939
    SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
    colDataAppendNNULL(pCol, 0, pSource->info.rows);
  }
2940

5
54liuyao 已提交
2941
  pDest->info.rows = pSource->info.rows;
2942 2943
  pDest->info.groupId = pSource->info.groupId;
  pDest->info.type = pSource->info.type;
5
54liuyao 已提交
2944 2945 2946
  blockDataUpdateTsWindow(pDest, 0);
}

5
54liuyao 已提交
2947 2948 2949 2950 2951 2952
static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
  clearSpecialDataBlock(pBlock);
  int32_t size = taosArrayGetSize(array);
  if (size - (*pIndex) == 0) {
    return;
  }
L
Liu Jicong 已提交
2953
  blockDataEnsureCapacity(pBlock, size - (*pIndex));
5
54liuyao 已提交
2954
  ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
2955 2956 2957 2958 2959
  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
5
54liuyao 已提交
2960
  for (; (*pIndex) < size; (*pIndex)++) {
L
Liu Jicong 已提交
2961
    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));
5
54liuyao 已提交
2962 2963 2964
    colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
    colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
2965 2966
    colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
5
54liuyao 已提交
2967 2968 2969 2970 2971 2972 2973 2974 2975
    pBlock->info.rows++;
  }
  if ((*pIndex) == size) {
    *pIndex = 0;
    taosArrayClear(array);
  }
  blockDataUpdateTsWindow(pBlock, 0);
}

L
Liu Jicong 已提交
2976
void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
5
54liuyao 已提交
2977
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
L
Liu Jicong 已提交
2978
  TSKEY*           tsData = (TSKEY*)pStartCol->pData;
5
54liuyao 已提交
2979
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
L
Liu Jicong 已提交
2980 2981
  uint64_t*        groupIdData = (uint64_t*)pGroupCol->pData;
  int32_t          chId = getChildIndex(pBlock);
5
54liuyao 已提交
2982
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
H
Haojun Liao 已提交
2983 2984
    SWinKey winRes = {.ts = tsData[i], .groupId = groupIdData[i]};
    void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
2985
    if (chIds) {
L
Liu Jicong 已提交
2986
      SArray* chArray = *(SArray**)chIds;
5
54liuyao 已提交
2987 2988
      int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
      if (index != -1) {
5
54liuyao 已提交
2989
        qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId);
5
54liuyao 已提交
2990 2991 2992
        taosArrayRemove(chArray, index);
        if (taosArrayGetSize(chArray) == 0) {
          // pull data is over
H
Haojun Liao 已提交
2993
          taosHashRemove(pMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
2994 2995 2996 2997 2998
        }
      }
    }
  }
}
5
54liuyao 已提交
2999

3000 3001 3002
static void addRetriveWindow(SArray* wins, SStreamFinalIntervalOperatorInfo* pInfo) {
  int32_t size = taosArrayGetSize(wins);
  for (int32_t i = 0; i < size; i++) {
L
Liu Jicong 已提交
3003
    SWinKey*    winKey = taosArrayGet(wins, i);
3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018
    STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
    if (isCloseWindow(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) {
      void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
      if (!chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId};
        // add pull data request
        savePullWindow(&pull, pInfo->pPullWins);
        int32_t size = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, winKey, size);
        qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, size);
      }
    }
  }
}

5
54liuyao 已提交
3019 3020 3021 3022 3023 3024
static void clearFunctionContext(SExprSupp* pSup) {
  for (int32_t i = 0; i < pSup->numOfExprs; i++) {
    pSup->pCtx[i].saveHandle.currentPage = -1;
  }
}

5
54liuyao 已提交
3025 3026
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
  SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
3027 3028 3029 3030 3031 3032

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  SArray*        pUpdated = taosArrayInit(4, POINTER_BYTES);
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
  TSKEY          maxTs = INT64_MIN;
5
54liuyao 已提交
3033

3034 3035
  SExprSupp* pSup = &pOperator->exprSupp;

5
54liuyao 已提交
3036
  qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
L
Liu Jicong 已提交
3037

5
54liuyao 已提交
3038 3039 3040
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
3041 3042 3043 3044
    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
    if (pInfo->pPullDataRes->info.rows != 0) {
      // process the rest of the data
      ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3045
      printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3046 3047 3048
      return pInfo->pPullDataRes;
    }

3049 3050 3051 3052 3053 3054 3055
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
      return pInfo->pDelRes;
    }

5
54liuyao 已提交
3056
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3057
    if (pInfo->binfo.pRes->info.rows == 0) {
5
54liuyao 已提交
3058
      pOperator->status = OP_EXEC_DONE;
5
54liuyao 已提交
3059
      if (!IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
3060
        clearFunctionContext(&pOperator->exprSupp);
5
54liuyao 已提交
3061 3062
        // semi interval operator clear disk buffer
        clearStreamIntervalOperator(pInfo);
3063
        qDebug("===stream===clear semi operator");
3064 3065
      } else {
        freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3066
      }
5
54liuyao 已提交
3067 3068
      return NULL;
    }
5
54liuyao 已提交
3069
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3070 3071
    return pInfo->binfo.pRes;
  } else {
5
54liuyao 已提交
3072 3073 3074
    if (!IS_FINAL_OP(pInfo)) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
      if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3075
        printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3076 3077
        return pInfo->binfo.pRes;
      }
5
54liuyao 已提交
3078 3079 3080 3081
    }
    if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
      pInfo->returnUpdate = false;
      ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3082
      printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3083 3084
      // process the rest of the data
      return pInfo->pUpdateRes;
5
54liuyao 已提交
3085
    }
3086 3087 3088
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
5
54liuyao 已提交
3089
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
3090 3091
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
3092 3093 3094 3095 3096
  }

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
3097
      clearSpecialDataBlock(pInfo->pUpdateRes);
5
54liuyao 已提交
3098
      removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
5
54liuyao 已提交
3099
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
3100
      qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3101 3102
      break;
    }
5
54liuyao 已提交
3103
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv");
5
54liuyao 已提交
3104
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
3105
    maxTs = TMAX(maxTs, pBlock->info.watermark);
3106

H
Haojun Liao 已提交
3107 3108
    ASSERT(pBlock->info.type != STREAM_INVERT);
    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
5
54liuyao 已提交
3109 3110
      pInfo->binfo.pRes->info.type = pBlock->info.type;
    } else if (pBlock->info.type == STREAM_CLEAR) {
H
Haojun Liao 已提交
3111
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinKey));
3112
      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
5
54liuyao 已提交
3113
      if (IS_FINAL_OP(pInfo)) {
L
Liu Jicong 已提交
3114 3115
        int32_t                           childIndex = getChildIndex(pBlock);
        SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
3116
        SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
L
Liu Jicong 已提交
3117
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;
3118

3119 3120 3121
        doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildSup->numOfExprs, pBlock, NULL);
        rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs,
                              pOperator->pTaskInfo, NULL);
5
54liuyao 已提交
3122 3123
        taosArrayDestroy(pUpWins);
        continue;
3124
      }
5
54liuyao 已提交
3125
      removeResults(pUpWins, pUpdatedMap);
3126
      copyDataBlock(pInfo->pUpdateRes, pBlock);
5
54liuyao 已提交
3127
      pInfo->returnUpdate = true;
3128
      taosArrayDestroy(pUpWins);
5
54liuyao 已提交
3129
      break;
3130
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
3131 3132
      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, delWins, &pInfo->interval, pUpdatedMap);
3133 3134 3135 3136 3137
      if (IS_FINAL_OP(pInfo)) {
        int32_t                           childIndex = getChildIndex(pBlock);
        SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;
3138
        doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval, NULL);
L
Liu Jicong 已提交
3139 3140
        rebuildIntervalWindow(pInfo, pSup, delWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs,
                              pOperator->pTaskInfo, pUpdatedMap);
3141 3142 3143
        addRetriveWindow(delWins, pInfo);
        taosArrayAddAll(pInfo->pDelWins, delWins);
        taosArrayDestroy(delWins);
3144 3145
        continue;
      }
3146 3147 3148
      removeResults(delWins, pUpdatedMap);
      taosArrayAddAll(pInfo->pDelWins, delWins);
      taosArrayDestroy(delWins);
3149
      break;
5
54liuyao 已提交
3150
    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
3151
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
5
54liuyao 已提交
3152
      continue;
5
54liuyao 已提交
3153
    } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
H
Haojun Liao 已提交
3154
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinKey));
5
54liuyao 已提交
3155
      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
5
54liuyao 已提交
3156
      removeResults(pUpWins, pUpdatedMap);
5
54liuyao 已提交
3157 3158 3159 3160 3161
      taosArrayDestroy(pUpWins);
      if (taosArrayGetSize(pUpdated) > 0) {
        break;
      }
      continue;
L
Liu Jicong 已提交
3162 3163
    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) {
      processPullOver(pBlock, pInfo->pPullDataMap);
5
54liuyao 已提交
3164
      continue;
5
54liuyao 已提交
3165
    }
5
54liuyao 已提交
3166

5
54liuyao 已提交
3167 3168 3169 3170
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
3171
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
3172
    doHashIntervalAgg(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
5
54liuyao 已提交
3173
    if (IS_FINAL_OP(pInfo)) {
S
shenglian zhou 已提交
3174
      int32_t chIndex = getChildIndex(pBlock);
5
54liuyao 已提交
3175 3176 3177 3178 3179
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
        SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
        if (!pChildOp) {
3180
          T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3181
        }
3182 3183
        SStreamFinalIntervalOperatorInfo* pTmpInfo = pChildOp->info;
        pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3184
        taosArrayPush(pInfo->pChildren, &pChildOp);
5
54liuyao 已提交
3185
        qDebug("===stream===add child, id:%d", chIndex);
5
54liuyao 已提交
3186
      }
S
shenglian zhou 已提交
3187
      SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
3188
      SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
3189
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
3190
      doHashIntervalAgg(pChildOp, pBlock, pBlock->info.groupId, NULL);
5
54liuyao 已提交
3191 3192
    }
  }
S
shenglian zhou 已提交
3193

5
54liuyao 已提交
3194
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
3195
  if (IS_FINAL_OP(pInfo)) {
3196
    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap,
5
54liuyao 已提交
3197
                        pUpdatedMap, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3198
    closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
3199 3200
  } else {
    pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;
5
54liuyao 已提交
3201 3202
  }

5
54liuyao 已提交
3203 3204 3205 3206 3207 3208 3209
  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosHashCleanup(pUpdatedMap);
  taosArraySort(pUpdated, resultrowComparAsc);

3210
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
3211 3212
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
3213 3214 3215 3216 3217

  doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
  if (pInfo->pPullDataRes->info.rows != 0) {
    // process the rest of the data
    ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3218
    printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3219 3220 3221
    return pInfo->pPullDataRes;
  }

3222
  // we should send result first.
5
54liuyao 已提交
3223
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3224
  if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3225
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3226 3227 3228 3229 3230 3231
    return pInfo->binfo.pRes;
  }

  if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
    pInfo->returnUpdate = false;
    ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3232
    printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3233 3234 3235
    // process the rest of the data
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
3236

3237 3238 3239
  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows != 0) {
    // process the rest of the data
5
54liuyao 已提交
3240
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
3241 3242
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
3243 3244 3245
  return NULL;
}

S
shenglian zhou 已提交
3246 3247 3248
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                     SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
  SIntervalPhysiNode*               pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
5
54liuyao 已提交
3249
  SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo));
S
shenglian zhou 已提交
3250
  SOperatorInfo*                    pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3251 3252 3253
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
3254

3255
  pOperator->pTaskInfo = pTaskInfo;
5
54liuyao 已提交
3256
  pInfo->order = TSDB_ORDER_ASC;
S
shenglian zhou 已提交
3257 3258 3259 3260 3261 3262 3263 3264
  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                .sliding = pIntervalPhyNode->sliding,
                                .intervalUnit = pIntervalPhyNode->intervalUnit,
                                .slidingUnit = pIntervalPhyNode->slidingUnit,
                                .offset = pIntervalPhyNode->offset,
                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pIntervalPhyNode->window.watermark,
5
54liuyao 已提交
3265 3266
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
S
shenglian zhou 已提交
3267
  };
3268
  ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY);
5
54liuyao 已提交
3269 3270
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
3271
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3272 3273 3274 3275 3276 3277 3278 3279 3280
  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

S
shenglian zhou 已提交
3281 3282
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5
54liuyao 已提交
3283
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
3284 3285

  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
3286 3287 3288 3289
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
3290
  initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs);
3291 3292
  initBasicInfo(&pInfo->binfo, pResBlock);

3293
  ASSERT(numOfCols > 0);
5
54liuyao 已提交
3294
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
3295

3296
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3297 3298
  pInfo->pChildren = NULL;
  if (numOfChild > 0) {
3299
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
5
54liuyao 已提交
3300 3301 3302
    for (int32_t i = 0; i < numOfChild; i++) {
      SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp) {
3303 3304
        SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
        pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3305 3306 3307 3308 3309 3310
        taosArrayPush(pInfo->pChildren, &pChildOp);
        continue;
      }
      goto _error;
    }
  }
3311
  pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
3312
  blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
5
54liuyao 已提交
3313 3314
  pInfo->returnUpdate = false;

3315
  pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
5
54liuyao 已提交
3316

5
54liuyao 已提交
3317 3318 3319 3320
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
    pInfo->isFinal = true;
    pOperator->name = "StreamFinalIntervalOperator";
  } else {
5
54liuyao 已提交
3321
    // semi interval operator does not catch result
5
54liuyao 已提交
3322 3323
    pInfo->isFinal = false;
    pOperator->name = "StreamSemiIntervalOperator";
H
Haojun Liao 已提交
3324
    ASSERT(pInfo->aggSup.currentPageId == -1);
5
54liuyao 已提交
3325 3326
  }

5
54liuyao 已提交
3327
  if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
5
54liuyao 已提交
3328 3329
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }
5
54liuyao 已提交
3330 3331 3332 3333
  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
  pInfo->pullIndex = 0;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
3334
  pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
5
54liuyao 已提交
3335
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
3336
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3337
  pInfo->delIndex = 0;
H
Haojun Liao 已提交
3338
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
5
54liuyao 已提交
3339
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
5
54liuyao 已提交
3340

5
54liuyao 已提交
3341
  pOperator->operatorType = pPhyNode->type;
5
54liuyao 已提交
3342 3343 3344 3345
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;

S
shenglian zhou 已提交
3346 3347 3348
  pOperator->fpSet =
      createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
3349
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
3350
    initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
3351
  }
5
54liuyao 已提交
3352 3353 3354 3355 3356 3357 3358 3359
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
3360
  destroyStreamFinalIntervalOperatorInfo(pInfo);
5
54liuyao 已提交
3361 3362 3363
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
5
54liuyao 已提交
3364
}
5
54liuyao 已提交
3365 3366 3367

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
3368
  void** pIte = NULL;
3369
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
3370
    SArray* pWins = (SArray*)(*pIte);
3371 3372 3373
    taosArrayDestroy(pWins);
  }
  taosHashCleanup(pSup->pResultRows);
5
54liuyao 已提交
3374
  destroyDiskbasedBuf(pSup->pResultBuf);
3375
  blockDataDestroy(pSup->pScanBlock);
5
54liuyao 已提交
3376 3377
}

5
54liuyao 已提交
3378 3379 3380 3381
void destroyStateWinInfo(void* ptr) {
  if (ptr == NULL) {
    return;
  }
3382
  SStateWindowInfo* pWin = (SStateWindowInfo*)ptr;
5
54liuyao 已提交
3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397
  taosMemoryFreeClear(pWin->stateKey.pData);
}

void destroyStateStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
  void** pIte = NULL;
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
    SArray* pWins = (SArray*)(*pIte);
    taosArrayDestroyEx(pWins, (FDelete)destroyStateWinInfo);
  }
  taosHashCleanup(pSup->pResultRows);
  destroyDiskbasedBuf(pSup->pResultBuf);
  blockDataDestroy(pSup->pScanBlock);
}

3398
void destroyStreamSessionAggOperatorInfo(void* param) {
5
54liuyao 已提交
3399
  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;
3400
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
3401 3402
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
3403 3404 3405
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
3406
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
3407
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
3408
      destroyStreamSessionAggOperatorInfo(pChInfo);
3409 3410 3411
      taosMemoryFreeClear(pChild);
    }
  }
5
54liuyao 已提交
3412 3413 3414 3415
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  blockDataDestroy(pInfo->pWinBlock);
  blockDataDestroy(pInfo->pUpdateRes);
5
54liuyao 已提交
3416 3417
  destroySqlFunctionCtx(pInfo->pDummyCtx, 0);
  taosHashCleanup(pInfo->pStDeleted);
3418

D
dapan1121 已提交
3419
  taosMemoryFreeClear(param);
5
54liuyao 已提交
3420 3421
}

3422 3423
int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,
                        SSDataBlock* pResultBlock) {
3424 3425 3426 3427
  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
  if (code != TSDB_CODE_SUCCESS) {
    return code;
  }
H
Haojun Liao 已提交
3428
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
3429

3430
  initBasicInfo(pBasicInfo, pResultBlock);
3431

5
54liuyao 已提交
3432
  for (int32_t i = 0; i < numOfCols; ++i) {
3433
    pSup->pCtx[i].saveHandle.pBuf = NULL;
5
54liuyao 已提交
3434
  }
3435

3436
  ASSERT(numOfCols > 0);
5
54liuyao 已提交
3437 3438 3439 3440 3441 3442 3443 3444
  return TSDB_CODE_SUCCESS;
}

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {
  for (int i = 0; i < nums; i++) {
    pDummy[i].functionId = pCtx[i].functionId;
  }
}
5
54liuyao 已提交
3445

X
Xiaoyu Wang 已提交
3446
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
3447 3448 3449 3450 3451 3452 3453 3454 3455 3456
                    uint16_t type, int32_t tsColIndex) {
  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
    pScanInfo->tsColIndex = tsColIndex;
  }

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
    initDownStream(downstream->pDownstream[0], pAggSup, gap, waterMark, type, tsColIndex);
    return;
  }
3457
  SStreamScanInfo* pScanInfo = downstream->info;
3458
  pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
5
54liuyao 已提交
3459
  pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
5
54liuyao 已提交
3460 3461
}

3462 3463
int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx,
                                int32_t numOfOutput) {
3464 3465 3466
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo));
}

3467 3468 3469 3470 3471 3472 3473
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                  SExecTaskInfo* pTaskInfo) {
  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
  int32_t                        numOfCols = 0;
  SExprInfo*                     pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock*                   pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;
X
Xiaoyu Wang 已提交
3474
  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
3475
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3476 3477 3478 3479
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

3480 3481
  pOperator->pTaskInfo = pTaskInfo;

3482
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3483 3484 3485
  if (pSessionNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar);
3486
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
5
54liuyao 已提交
3487 3488 3489 3490
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
3491
  SExprSupp* pSup = &pOperator->exprSupp;
5
54liuyao 已提交
3492

3493
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
3494 3495 3496
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
3497

3498
  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
3499 3500 3501
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
X
Xiaoyu Wang 已提交
3502

5
54liuyao 已提交
3503 3504 3505 3506
  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }
3507
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
3508

3509 3510
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType, .maxTs = INT64_MIN};
H
Haojun Liao 已提交
3511 3512

  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3513 3514
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

5
54liuyao 已提交
3515 3516 3517 3518
  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  if (pSessionNode->window.pTsEnd) {
    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;
  }
3519
  pInfo->gap = pSessionNode->gap;
5
54liuyao 已提交
3520 3521 3522 3523 3524
  pInfo->binfo.pRes = pResBlock;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
5
54liuyao 已提交
3525
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3526
  pInfo->pChildren = NULL;
5
54liuyao 已提交
3527 3528
  pInfo->isFinal = false;
  pInfo->pPhyNode = pPhyNode;
5
54liuyao 已提交
3529
  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
5
54liuyao 已提交
3530 3531

  pOperator->name = "StreamSessionWindowAggOperator";
3532
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
5
54liuyao 已提交
3533 3534 3535
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
3536 3537 3538
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
3539
  if (downstream) {
L
Liu Jicong 已提交
3540 3541
    initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType,
                   pInfo->primaryTsIndex);
5
54liuyao 已提交
3542 3543
    code = appendDownstream(pOperator, &downstream, 1);
  }
5
54liuyao 已提交
3544 3545 3546 3547
  return pOperator;

_error:
  if (pInfo != NULL) {
3548
    destroyStreamSessionAggOperatorInfo(pInfo);
5
54liuyao 已提交
3549 3550 3551 3552 3553 3554 3555 3556
  }

  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

int64_t getSessionWindowEndkey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
3557
  SArray*            pWinInfos = (SArray*)data;
5
54liuyao 已提交
3558 3559 3560
  SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index);
  return pWin->win.ekey;
}
5
54liuyao 已提交
3561 3562

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {
5
54liuyao 已提交
3563
  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {
5
54liuyao 已提交
3564 3565 3566 3567 3568
    return true;
  }
  return false;
}

3569
bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); }
5
54liuyao 已提交
3570

3571
static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs, int32_t index) {
L
Liu Jicong 已提交
3572 3573
  SResultWindowInfo win = {
      .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
5
54liuyao 已提交
3574 3575 3576
  return taosArrayInsert(pWinInfos, index, &win);
}

3577
static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs) {
L
Liu Jicong 已提交
3578 3579
  SResultWindowInfo win = {
      .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
5
54liuyao 已提交
3580 3581 3582
  return taosArrayPush(pWinInfos, &win);
}

3583
SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) {
3584
  void**  ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t));
3585 3586 3587
  SArray* pWinInfos = NULL;
  if (ite == NULL) {
    pWinInfos = taosArrayInit(1024, pAggSup->valueSize);
3588
    taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void*));
3589 3590 3591 3592 3593 3594
  } else {
    pWinInfos = *ite;
  }
  return pWinInfos;
}

5
54liuyao 已提交
3595 3596
// don't add new window
SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
3597
                                       int64_t gap, int32_t* pIndex) {
5
54liuyao 已提交
3598 3599 3600 3601 3602 3603 3604 3605
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    return NULL;
  }
  // find the first position which is smaller than the key
3606
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isInWindow(pWin, startTs, gap)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isInWindow(pWin, startTs, gap)) {
      *pIndex = index + 1;
      return pWin;
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
3622
      *pIndex = index + 1;
5
54liuyao 已提交
3623 3624 3625 3626 3627 3628 3629
      return pWin;
    }
  }

  return NULL;
}

3630 3631
SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
                                        int64_t gap, int32_t* pIndex) {
3632 3633 3634
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

5
54liuyao 已提交
3635 3636
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
5
54liuyao 已提交
3637
    *pIndex = 0;
3638
    return addNewSessionWindow(pWinInfos, startTs, endTs);
5
54liuyao 已提交
3639 3640
  }
  // find the first position which is smaller than the key
5
54liuyao 已提交
3641
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3642 3643 3644
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
5
54liuyao 已提交
3645
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3646 3647 3648 3649 3650 3651 3652
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
5
54liuyao 已提交
3653
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3654 3655
      *pIndex = index + 1;
      return pWin;
5
54liuyao 已提交
3656 3657 3658
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
      *pIndex = index;
      return pWin;
5
54liuyao 已提交
3659 3660 3661 3662 3663
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
3664
    return addNewSessionWindow(pWinInfos, startTs, endTs);
5
54liuyao 已提交
3665
  }
5
54liuyao 已提交
3666
  *pIndex = index + 1;
3667
  return insertNewSessionWindow(pWinInfos, startTs, endTs, index + 1);
5
54liuyao 已提交
3668 3669
}

dengyihao's avatar
dengyihao 已提交
3670 3671
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId,
                                int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted) {
5
54liuyao 已提交
3672
  for (int32_t i = start; i < rows; ++i) {
3673
    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {
5
54liuyao 已提交
3674 3675
      return i - start;
    }
5
54liuyao 已提交
3676
    if (pWinInfo->win.skey > pStartTs[i]) {
5
54liuyao 已提交
3677
      if (pStDeleted && pWinInfo->isOutput) {
H
Haojun Liao 已提交
3678
        SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
3679
        taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
3680 3681
        pWinInfo->isOutput = false;
      }
5
54liuyao 已提交
3682 3683 3684 3685 3686
      pWinInfo->win.skey = pStartTs[i];
    }
    pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pStartTs[i]);
    if (pEndTs) {
      pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pEndTs[i]);
5
54liuyao 已提交
3687 3688 3689 3690 3691
    }
  }
  return rows - start;
}

X
Xiaoyu Wang 已提交
3692
static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,
3693
                                  uint64_t groupId, int32_t numOfOutput, int32_t* rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3694
                                  SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) {
5
54liuyao 已提交
3695 3696
  assert(pWinInfo->win.skey <= pWinInfo->win.ekey);
  // too many time window in query
3697
  int32_t size = taosArrayGetSize(pAggSup->pCurWins);
3698
  if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH && size > MAX_INTERVAL_TIME_WINDOW) {
3699
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
5
54liuyao 已提交
3700
  }
X
Xiaoyu Wang 已提交
3701

5
54liuyao 已提交
3702
  if (pWinInfo->pos.pageId == -1) {
3703
    *pResult = getNewResultRow(pAggSup->pResultBuf, &pAggSup->currentPageId, pAggSup->resultRowSize);
5
54liuyao 已提交
3704 3705 3706 3707 3708 3709 3710 3711
    if (*pResult == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }

    // add a new result set for a new group
    pWinInfo->pos.pageId = (*pResult)->pageId;
    pWinInfo->pos.offset = (*pResult)->offset;
  } else {
3712
    *pResult = getResultRowByPos(pAggSup->pResultBuf, &pWinInfo->pos, true);
5
54liuyao 已提交
3713 3714 3715 3716 3717 3718 3719 3720
    if (!(*pResult)) {
      qError("getResultRowByPos return NULL, TID:%s", GET_TASKID(pTaskInfo));
      return TSDB_CODE_FAILED;
    }
  }

  // set time window for current result
  (*pResult)->win = pWinInfo->win;
3721
  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
5
54liuyao 已提交
3722 3723 3724
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3725 3726 3727
static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup,
                                  SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock,
                                  SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3728
                                  int32_t numOutput, SOperatorInfo* pOperator) {
3729
  SExprSupp*     pSup = &pOperator->exprSupp;
3730 3731
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

X
Xiaoyu Wang 已提交
3732 3733
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId);
  TSKEY*           tsCols = (int64_t*)pColDataInfo->pData;
3734 3735
  int32_t          code = setWindowOutputBuf(pCurWin, pResult, pSup->pCtx, pSDataBlock->info.groupId, numOutput,
                                             pSup->rowEntryInfoOffset, pAggSup, pTaskInfo);
5
54liuyao 已提交
3736 3737 3738
  if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }
5
54liuyao 已提交
3739
  updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false);
3740
  doApplyFunctions(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, pSDataBlock->info.rows, numOutput);
5
54liuyao 已提交
3741 3742 3743
  SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pCurWin->pos.pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pAggSup->pResultBuf, bufPage);
5
54liuyao 已提交
3744 3745 3746
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3747 3748
static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                              SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3749
                              int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3750
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3751
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3752 3753
}

X
Xiaoyu Wang 已提交
3754 3755
static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                                   SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex,
3756
                                   int32_t winRows, int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3757
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3758
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3759 3760
}

5
54liuyao 已提交
3761 3762
int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) {
  SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex);
X
Xiaoyu Wang 已提交
3763
  int32_t            size = taosArrayGetSize(pWinInfos);
5
54liuyao 已提交
3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i);
    if (!isInWindow(pCurWin, pWinInfo->win.skey, gap)) {
      return i - startIndex - 1;
    }
  }

  return size - startIndex - 1;
}

5
54liuyao 已提交
3775
void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId,
3776
                       int32_t numOfOutput, SHashObj* pStUpdated, SHashObj* pStDeleted, SOperatorInfo* pOperator) {
3777
  SExprSupp*     pSup = &pOperator->exprSupp;
3778 3779
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

3780
  SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex);
X
Xiaoyu Wang 已提交
3781
  SResultRow*        pCurResult = NULL;
3782
  setWindowOutputBuf(pCurWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3783
                     &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3784
  num += startIndex + 1;
3785
  ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins));
5
54liuyao 已提交
3786 3787
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < num; i++) {
3788
    SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i);
X
Xiaoyu Wang 已提交
3789
    SResultRow*        pWinResult = NULL;
3790
    setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3791
                       &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3792
    pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey);
3793 3794
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true);
    compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
3795 3796
    taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition));
    if (pWinInfo->isOutput) {
H
Haojun Liao 已提交
3797
      SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
3798
      taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
3799 3800
      pWinInfo->isOutput = false;
    }
3801
    taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
5
54liuyao 已提交
3802 3803
    SFilePage* tmpPage = getBufPage(pInfo->streamAggSup.pResultBuf, pWinInfo->pos.pageId);
    releaseBufPage(pInfo->streamAggSup.pResultBuf, tmpPage);
5
54liuyao 已提交
3804
  }
5
54liuyao 已提交
3805 3806 3807 3808
  SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pCurWin->pos.pageId);
  ASSERT(num > 0);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
5
54liuyao 已提交
3809 3810
}

3811 3812
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
                                   SHashObj* pStDeleted, bool hasEndTs) {
X
Xiaoyu Wang 已提交
3813
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3814
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3815
  bool                           masterScan = true;
3816
  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
5
54liuyao 已提交
3817
  uint64_t                       groupId = pSDataBlock->info.groupId;
X
Xiaoyu Wang 已提交
3818 3819 3820 3821 3822
  int64_t                        gap = pInfo->gap;
  int64_t                        code = TSDB_CODE_SUCCESS;

  int32_t     step = 1;
  bool        ascScan = true;
5
54liuyao 已提交
3823 3824
  TSKEY*      startTsCols = NULL;
  TSKEY*      endTsCols = NULL;
5
54liuyao 已提交
3825
  SResultRow* pResult = NULL;
X
Xiaoyu Wang 已提交
3826
  int32_t     winRows = 0;
5
54liuyao 已提交
3827

5
54liuyao 已提交
3828 3829 3830 3831 3832 3833
  ASSERT(pSDataBlock->pDataBlock);
  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  startTsCols = (int64_t*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = NULL;
  if (hasEndTs) {
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);
5
54liuyao 已提交
3834
  } else {
5
54liuyao 已提交
3835
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
3836
  }
5
54liuyao 已提交
3837
  endTsCols = (int64_t*)pEndTsCol->pData;
X
Xiaoyu Wang 已提交
3838

5
54liuyao 已提交
3839
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
3840
  for (int32_t i = 0; i < pSDataBlock->info.rows;) {
5
54liuyao 已提交
3841
    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
3842 3843 3844
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
3845
    int32_t            winIndex = 0;
3846
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex);
dengyihao's avatar
dengyihao 已提交
3847 3848
    winRows = updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, groupId, pSDataBlock->info.rows, i, pInfo->gap,
                                      pStDeleted);
3849
    code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
3850
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
3851
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3852
    }
5
54liuyao 已提交
3853

3854
    int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap);
5
54liuyao 已提交
3855
    if (winNum > 0) {
3856
      compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, pStDeleted, pOperator);
5
54liuyao 已提交
3857
    }
5
54liuyao 已提交
3858
    pCurWin->isClosed = false;
5
54liuyao 已提交
3859
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
H
Haojun Liao 已提交
3860 3861
      SWinKey value = {.ts = pCurWin->win.skey, .groupId = groupId};
      code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
5
54liuyao 已提交
3862
      if (code != TSDB_CODE_SUCCESS) {
3863
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3864 3865
      }
      pCurWin->isOutput = true;
5
54liuyao 已提交
3866 3867 3868 3869 3870
    }
    i += winRows;
  }
}

5
54liuyao 已提交
3871
void deleteWindow(SArray* pWinInfos, int32_t index, FDelete fp) {
5
54liuyao 已提交
3872
  ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos));
5
54liuyao 已提交
3873 3874 3875 3876
  if (fp) {
    void* ptr = taosArrayGet(pWinInfos, index);
    fp(ptr);
  }
5
54liuyao 已提交
3877 3878 3879
  taosArrayRemove(pWinInfos, index);
}

3880 3881
static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result,
                                FDelete fp) {
5
54liuyao 已提交
3882
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
3883
  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;
5
54liuyao 已提交
3884
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
3885
  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;
3886
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
3887
  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;
5
54liuyao 已提交
3888 3889
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    int32_t winIndex = 0;
3890 3891
    while (1) {
      SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex);
5
54liuyao 已提交
3892 3893 3894
      if (!pCurWin) {
        break;
      }
3895
      SResultWindowInfo delWin = *pCurWin;
5
54liuyao 已提交
3896
      deleteWindow(pAggSup->pCurWins, winIndex, fp);
5
54liuyao 已提交
3897
      if (result) {
3898 3899
        delWin.groupId = gpDatas[i];
        taosArrayPush(result, &delWin);
5
54liuyao 已提交
3900 3901 3902 3903 3904
      }
    }
  }
}

3905 3906
static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock, int32_t tsIndex,
                                  int32_t numOfOutput, int64_t gap, SArray* result) {
5
54liuyao 已提交
3907
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
3908
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
H
Haojun Liao 已提交
3909 3910
  SColumnInfoData* pGpDataInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*        gpCols = (uint64_t*)pGpDataInfo->pData;
X
Xiaoyu Wang 已提交
3911
  int32_t          step = 0;
5
54liuyao 已提交
3912
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
3913
    int32_t            winIndex = 0;
H
Haojun Liao 已提交
3914
    SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, tsCols[i], INT64_MIN, gpCols[i], gap, &winIndex);
5
54liuyao 已提交
3915
    if (!pCurWin || pCurWin->pos.pageId == -1) {
5
54liuyao 已提交
3916
      // window has been closed.
5
54liuyao 已提交
3917
      step = 1;
5
54liuyao 已提交
3918 3919
      continue;
    }
5
54liuyao 已提交
3920
    step = updateSessionWindowInfo(pCurWin, tsCols, NULL, 0, pBlock->info.rows, i, gap, NULL);
5
54liuyao 已提交
3921
    ASSERT(isInWindow(pCurWin, tsCols[i], gap));
3922
    doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
3923
    if (result) {
3924
      pCurWin->groupId = gpCols[i];
3925 3926
      taosArrayPush(result, pCurWin);
    }
5
54liuyao 已提交
3927 3928 3929
  }
}

5
54liuyao 已提交
3930
static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) {
X
Xiaoyu Wang 已提交
3931
  void*  pData = NULL;
5
54liuyao 已提交
3932
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
3933
  while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) {
5
54liuyao 已提交
3934 3935 3936 3937 3938 3939
    void* key = taosHashGetKey(pData, &keyLen);
    ASSERT(keyLen == sizeof(SResultRowPosition));
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    if (pos == NULL) {
      return TSDB_CODE_QRY_OUT_OF_MEMORY;
    }
H
Haojun Liao 已提交
3940
    pos->groupId = ((SWinKey*)pData)->groupId;
5
54liuyao 已提交
3941
    pos->pos = *(SResultRowPosition*)key;
H
Haojun Liao 已提交
3942
    *(int64_t*)pos->key = ((SWinKey*)pData)->ts;
5
54liuyao 已提交
3943 3944
    taosArrayPush(pUpdated, &pos);
  }
5
54liuyao 已提交
3945
  taosArraySort(pUpdated, resultrowComparAsc);
5
54liuyao 已提交
3946 3947 3948 3949 3950
  return TSDB_CODE_SUCCESS;
}

void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
  blockDataCleanup(pBlock);
3951 3952 3953 3954 3955
  int32_t size = taosHashGetSize(pStDeleted);
  if (size == 0) {
    return;
  }
  blockDataEnsureCapacity(pBlock, size);
5
54liuyao 已提交
3956
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
3957
  while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
H
Haojun Liao 已提交
3958
    SWinKey*         res = *Ite;
3959 3960 3961 3962 3963 3964
    SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
    colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->ts, false);
    SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
    colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->ts, false);
    SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
    colDataAppendNULL(pUidCol, pBlock->info.rows);
5
54liuyao 已提交
3965 3966
    SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
    colDataAppend(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false);
3967 3968 3969 3970
    SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
    colDataAppendNULL(pCalStCol, pBlock->info.rows);
    SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
    colDataAppendNULL(pCalEdCol, pBlock->info.rows);
5
54liuyao 已提交
3971 3972 3973 3974 3975 3976 3977 3978 3979 3980
    pBlock->info.rows += 1;
    if (pBlock->info.rows + 1 >= pBlock->info.capacity) {
      break;
    }
  }
  if ((*Ite) == NULL) {
    taosHashClear(pStDeleted);
  }
}

L
Liu Jicong 已提交
3981 3982
static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t numOfOutput,
                              SOperatorInfo* pOperator, SHashObj* pStUpdated, bool needCreate) {
3983
  SExprSupp*     pSup = &pOperator->exprSupp;
3984 3985
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

3986 3987
  int32_t size = taosArrayGetSize(pWinArray);
  ASSERT(pInfo->pChildren);
3988

3989 3990
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i);
X
Xiaoyu Wang 已提交
3991
    SResultRow*        pCurResult = NULL;
3992 3993 3994
    uint64_t           groupId = pParentWin->groupId;
    int32_t            winIndex = 0;
    if (needCreate) {
L
Liu Jicong 已提交
3995 3996
      pParentWin =
          getSessionTimeWindow(&pInfo->streamAggSup, pParentWin->win.skey, pParentWin->win.ekey, groupId, 0, &winIndex);
3997
    }
3998
    setWindowOutputBuf(pParentWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3999
                       &pInfo->streamAggSup, pTaskInfo);
4000
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
4001
    int32_t num = 0;
4002
    for (int32_t j = 0; j < numOfChildren; j++) {
X
Xiaoyu Wang 已提交
4003
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
4004
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
4005
      SArray*                        pChWins = getWinInfos(&pChInfo->streamAggSup, groupId);
X
Xiaoyu Wang 已提交
4006 4007
      int32_t                        chWinSize = taosArrayGetSize(pChWins);
      int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
4008
      if (index < 0) {
4009
        index = 0;
5
54liuyao 已提交
4010 4011
      }
      for (int32_t k = index; k < chWinSize; k++) {
5
54liuyao 已提交
4012 4013
        SResultWindowInfo* pChWin = taosArrayGet(pChWins, k);
        if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) {
4014
          SResultRow* pChResult = NULL;
5
54liuyao 已提交
4015
          setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
4016
                             pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
4017 4018
          updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pChWin->win, true);
          compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
4019 4020
          SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
          releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage);
4021
          num++;
4022
          continue;
5
54liuyao 已提交
4023 4024
        } else if (!pChWin->isClosed) {
          break;
4025 4026 4027
        }
      }
    }
4028 4029 4030 4031 4032 4033 4034
    if (num == 0 && needCreate) {
      deleteWindow(pInfo->streamAggSup.pCurWins, winIndex, NULL);
    }
    if (pStUpdated && num > 0) {
      SWinKey value = {.ts = pParentWin->win.skey, .groupId = groupId};
      taosHashPut(pStUpdated, &pParentWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
    }
5
54liuyao 已提交
4035 4036 4037 4038
    SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId);
    ASSERT(size > 0);
    setBufPageDirty(bufPage, true);
    releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
4039 4040 4041
  }
}

X
Xiaoyu Wang 已提交
4042
typedef SResultWindowInfo* (*__get_win_info_)(void*);
5
54liuyao 已提交
4043 4044
SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
5
54liuyao 已提交
4045

4046
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn,
5
54liuyao 已提交
4047
                           bool delete, FDelete fp) {
5
54liuyao 已提交
4048
  // Todo(liuyao) save window to tdb
4049
  void** pIte = NULL;
5
54liuyao 已提交
4050
  size_t keyLen = 0;
4051
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
5
54liuyao 已提交
4052
    uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen);
4053 4054
    SArray*   pWins = (SArray*)(*pIte);
    int32_t   size = taosArrayGetSize(pWins);
4055 4056 4057
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
5
54liuyao 已提交
4058
      if (isCloseWindow(&pSeWin->win, pTwSup)) {
4059 4060
        if (!pSeWin->isClosed) {
          pSeWin->isClosed = true;
5
54liuyao 已提交
4061
          if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {
5
54liuyao 已提交
4062
            int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed);
5
54liuyao 已提交
4063 4064 4065
            if (code != TSDB_CODE_SUCCESS) {
              return code;
            }
4066 4067
            pSeWin->isOutput = true;
          }
5
54liuyao 已提交
4068
          if (delete) {
5
54liuyao 已提交
4069
            deleteWindow(pWins, i, fp);
5
54liuyao 已提交
4070 4071 4072
            i--;
            size = taosArrayGetSize(pWins);
          }
5
54liuyao 已提交
4073
        }
4074
        continue;
5
54liuyao 已提交
4075
      }
4076
      break;
5
54liuyao 已提交
4077 4078 4079 4080 4081
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
4082
static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs, bool delete, FDelete fp) {
5
54liuyao 已提交
4083 4084 4085 4086 4087
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);
    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4088
    closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL, getResWinForSession, delete, fp);
5
54liuyao 已提交
4089 4090 4091
  }
}

4092
int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) {
4093
  void** pIte = NULL;
4094
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
4095
    SArray* pWins = (SArray*)(*pIte);
4096 4097 4098 4099 4100 4101 4102 4103
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (!pSeWin->isClosed) {
        int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed);
        pSeWin->isOutput = true;
      }
5
54liuyao 已提交
4104 4105 4106 4107 4108
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
4109 4110 4111 4112
static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) {
  int32_t size = taosArrayGetSize(pResWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i);
H
Haojun Liao 已提交
4113
    SWinKey            res = {.ts = pWinInfo->win.skey, .groupId = pWinInfo->groupId};
4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126
    taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
  }
}

static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) {
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWin = taosArrayGet(pWins, i);
    taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition));
  }
}

int32_t compareWinKey(void* pKey, void* data, int32_t index) {
L
Liu Jicong 已提交
4127
  SArray*     res = (SArray*)data;
4128
  SResKeyPos* pos = taosArrayGetP(res, index);
L
Liu Jicong 已提交
4129
  SWinKey*    pData = (SWinKey*)pKey;
4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151
  if (pData->ts == *(int64_t*)pos->key) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->ts > *(int64_t*)pos->key) {
    return 1;
  }
  return -1;
}

static void removeSessionDeleteResults(SArray* update, SHashObj* pStDeleted) {
  int32_t size = taosHashGetSize(pStDeleted);
  if (size == 0) {
    return;
  }

  int32_t num = taosArrayGetSize(update);
  for (int32_t i = 0; i < num; i++) {
    SResKeyPos* pos = taosArrayGetP(update, i);
L
Liu Jicong 已提交
4152
    SWinKey     winKey = {.ts = *(int64_t*)pos->key, .groupId = pos->groupId};
4153
    taosHashRemove(pStDeleted, &winKey, sizeof(SWinKey));
5
54liuyao 已提交
4154 4155 4156
  }
}

5
54liuyao 已提交
4157
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
5
54liuyao 已提交
4158
  SExprSupp*                     pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4159
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4160
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
5
54liuyao 已提交
4161 4162 4163 4164
  TSKEY                          maxTs = INT64_MIN;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
4165 4166
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4167
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4168 4169
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
4170
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
4171
    if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
5
54liuyao 已提交
4172 4173
      doSetOperatorCompleted(pOperator);
    }
5
54liuyao 已提交
4174
    printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4175 4176 4177
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
4178 4179
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
4180
  SOperatorInfo* downstream = pOperator->pDownstream[0];
L
Liu Jicong 已提交
4181
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);  // SResKeyPos
5
54liuyao 已提交
4182 4183 4184 4185 4186
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4187
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv");
4188

5
54liuyao 已提交
4189
    if (pBlock->info.type == STREAM_CLEAR) {
X
Xiaoyu Wang 已提交
4190
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
H
Haojun Liao 已提交
4191 4192
      doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, START_TS_COLUMN_INDEX,
                            pOperator->exprSupp.numOfExprs, 0, pWins);
5
54liuyao 已提交
4193 4194
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
X
Xiaoyu Wang 已提交
4195
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
4196
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
L
Liu Jicong 已提交
4197 4198
        doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX,
                              pChildOp->exprSupp.numOfExprs, 0, NULL);
4199
        rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, NULL, false);
4200 4201
      }
      taosArrayDestroy(pWins);
5
54liuyao 已提交
4202
      continue;
5
54liuyao 已提交
4203 4204 4205
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      // gap must be 0
5
54liuyao 已提交
4206
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL);
5
54liuyao 已提交
4207 4208 4209 4210 4211
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
        // gap must be 0
5
54liuyao 已提交
4212
        doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL);
4213
        rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated, true);
5
54liuyao 已提交
4214 4215
      }
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
4216
      removeSessionResults(pStUpdated, pWins);
5
54liuyao 已提交
4217 4218
      taosArrayDestroy(pWins);
      continue;
4219
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4220
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
5
54liuyao 已提交
4221
      continue;
5
54liuyao 已提交
4222
    }
5
54liuyao 已提交
4223

5
54liuyao 已提交
4224 4225 4226 4227
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4228
    // the pDataBlock are always the same one, no need to call this again
4229
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4230 4231 4232 4233 4234 4235
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo));
    if (IS_FINAL_OP(pInfo)) {
      int32_t chIndex = getChildIndex(pBlock);
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
4236 4237
        SOperatorInfo* pChildOp =
            createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
5
54liuyao 已提交
4238
        if (!pChildOp) {
4239
          T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4240 4241 4242
        }
        taosArrayPush(pInfo->pChildren, &pChildOp);
      }
4243
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
4244 4245
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true);
4246
    }
5
54liuyao 已提交
4247
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
4248
    maxTs = TMAX(maxTs, pBlock->info.watermark);
5
54liuyao 已提交
4249
  }
5
54liuyao 已提交
4250 4251

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4252 4253
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
4254

4255
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession,
5
54liuyao 已提交
4256 4257
                     pInfo->ignoreExpiredData, NULL);
  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, NULL);
5
54liuyao 已提交
4258
  copyUpdateResult(pStUpdated, pUpdated);
4259
  removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
5
54liuyao 已提交
4260 4261
  taosHashCleanup(pStUpdated);

4262
  finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4263 4264 4265 4266
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4267
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4268 4269 4270
    return pInfo->pDelRes;
  }
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
4271
  printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4272 4273 4274 4275
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
4276
  void** pIte = NULL;
5
54liuyao 已提交
4277
  while ((pIte = taosHashIterate(pInfo->streamAggSup.pResultRows, pIte)) != NULL) {
4278
    SArray* pWins = (SArray*)(*pIte);
5
54liuyao 已提交
4279 4280 4281 4282 4283 4284 4285 4286
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      SResultWindowInfo* pWin = (SResultWindowInfo*)taosArrayGet(pWins, i);
      pWin->pos.pageId = -1;
      pWin->pos.offset = -1;
    }
  }
  clearDiskbasedBuf(pInfo->streamAggSup.pResultBuf);
4287
  pInfo->streamAggSup.currentPageId = -1;
5
54liuyao 已提交
4288 4289 4290 4291 4292 4293 4294
}

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
  TSKEY                          maxTs = INT64_MIN;
  SExprSupp*                     pSup = &pOperator->exprSupp;
4295

5
54liuyao 已提交
4296 4297
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
4298
  }
L
Liu Jicong 已提交
4299

4300
  {
5
54liuyao 已提交
4301 4302
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
    if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
4303
      printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
4304 4305 4306
      return pBInfo->pRes;
    }

4307 4308
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
H
Haojun Liao 已提交
4309
      printDataBlock(pInfo->pDelRes, "semi session");
5
54liuyao 已提交
4310 4311
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
4312

4313 4314
    if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
      pInfo->returnUpdate = false;
5
54liuyao 已提交
4315
      // process the rest of the data
H
Haojun Liao 已提交
4316
      printDataBlock(pInfo->pUpdateRes, "semi session");
5
54liuyao 已提交
4317 4318
      return pInfo->pUpdateRes;
    }
4319 4320

    if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
4321
      clearFunctionContext(&pOperator->exprSupp);
4322 4323 4324 4325 4326
      // semi interval operator clear disk buffer
      clearStreamSessionOperator(pInfo);
      pOperator->status = OP_EXEC_DONE;
      return NULL;
    }
5
54liuyao 已提交
4327 4328 4329 4330 4331 4332 4333 4334 4335
  }

  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  SOperatorInfo* downstream = pOperator->pDownstream[0];
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
4336
      clearSpecialDataBlock(pInfo->pUpdateRes);
4337
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
4338 4339
      break;
    }
H
Haojun Liao 已提交
4340
    printDataBlock(pBlock, "semi session recv");
5
54liuyao 已提交
4341

5
54liuyao 已提交
4342
    if (pBlock->info.type == STREAM_CLEAR) {
5
54liuyao 已提交
4343
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
H
Haojun Liao 已提交
4344
      doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, START_TS_COLUMN_INDEX, pSup->numOfExprs, 0, pWins);
5
54liuyao 已提交
4345 4346
      removeSessionResults(pStUpdated, pWins);
      taosArrayDestroy(pWins);
H
Haojun Liao 已提交
4347
      copyDataBlock(pInfo->pUpdateRes, pBlock);
4348
      pInfo->returnUpdate = true;
5
54liuyao 已提交
4349
      break;
5
54liuyao 已提交
4350 4351
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      // gap must be 0
4352 4353 4354 4355 4356
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL);
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
      removeSessionResults(pStUpdated, pWins);
      taosArrayDestroy(pWins);
5
54liuyao 已提交
4357
      break;
5
54liuyao 已提交
4358 4359 4360 4361 4362
    } else if (pBlock->info.type == STREAM_GET_ALL) {
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
      continue;
    }

5
54liuyao 已提交
4363 4364 4365 4366
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
5
54liuyao 已提交
4367 4368
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
4369
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, NULL, false);
5
54liuyao 已提交
4370 4371 4372 4373
    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
4374
  pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;
4375

5
54liuyao 已提交
4376
  copyUpdateResult(pStUpdated, pUpdated);
4377
  removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
5
54liuyao 已提交
4378
  taosHashCleanup(pStUpdated);
5
54liuyao 已提交
4379

4380 4381
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4382
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
5
54liuyao 已提交
4383
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
4384 4385 4386

  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
  if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
4387
    printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
4388 4389 4390
    return pBInfo->pRes;
  }

4391 4392
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
H
Haojun Liao 已提交
4393
    printDataBlock(pInfo->pDelRes, "semi session");
5
54liuyao 已提交
4394 4395
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
4396

4397 4398
  if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
    pInfo->returnUpdate = false;
5
54liuyao 已提交
4399
    // process the rest of the data
H
Haojun Liao 已提交
4400
    printDataBlock(pInfo->pUpdateRes, "semi session");
5
54liuyao 已提交
4401 4402
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
4403 4404 4405

  pOperator->status = OP_EXEC_DONE;
  return NULL;
5
54liuyao 已提交
4406
}
4407

4408 4409
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                       SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
4410 4411
  int32_t        code = TSDB_CODE_OUT_OF_MEMORY;
  SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo);
4412 4413 4414
  if (pOperator == NULL) {
    goto _error;
  }
4415
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
5
54liuyao 已提交
4416 4417 4418 4419 4420 4421

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
    pInfo->isFinal = true;
    pOperator->name = "StreamSessionFinalAggOperator";
  } else {
    pInfo->isFinal = false;
H
Haojun Liao 已提交
4422
    pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
4423 4424 4425
    blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
    pOperator->name = "StreamSessionSemiAggOperator";
    pOperator->fpSet =
4426 4427
        createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL,
                            destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
4428 4429 4430 4431 4432
  }
  pOperator->operatorType = pPhyNode->type;
  if (numOfChild > 0) {
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
    for (int32_t i = 0; i < numOfChild; i++) {
4433
      SOperatorInfo* pChild = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
5
54liuyao 已提交
4434 4435 4436 4437
      if (pChild == NULL) {
        goto _error;
      }
      taosArrayPush(pInfo->pChildren, &pChild);
4438 4439 4440 4441 4442 4443
    }
  }
  return pOperator;

_error:
  if (pInfo != NULL) {
4444
    destroyStreamSessionAggOperatorInfo(pInfo);
4445 4446 4447 4448 4449 4450
  }

  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5
54liuyao 已提交
4451

4452
void destroyStreamStateOperatorInfo(void* param) {
X
Xiaoyu Wang 已提交
4453
  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;
4454
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
4455
  destroyStateStreamAggSupporter(&pInfo->streamAggSup);
5
54liuyao 已提交
4456 4457 4458 4459
  cleanupGroupResInfo(&pInfo->groupResInfo);
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
4460
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
5
54liuyao 已提交
4461
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
4462
      destroyStreamSessionAggOperatorInfo(pChInfo);
5
54liuyao 已提交
4463 4464 4465 4466
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
5
54liuyao 已提交
4467 4468 4469 4470
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  taosHashCleanup(pInfo->pSeDeleted);
  destroySqlFunctionCtx(pInfo->pDummyCtx, 0);
D
dapan1121 已提交
4471 4472

  taosMemoryFreeClear(param);
5
54liuyao 已提交
4473 4474 4475 4476 4477 4478 4479
}

int64_t getStateWinTsKey(void* data, int32_t index) {
  SStateWindowInfo* pStateWin = taosArrayGet(data, index);
  return pStateWin->winInfo.win.ekey;
}

X
Xiaoyu Wang 已提交
4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491
SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4492 4493 4494 4495 4496 4497 4498 4499
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayPush(pWinInfos, &win);
}

X
Xiaoyu Wang 已提交
4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511
SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayInsert(pWinInfos, index, &win);
}

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {
  if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) {
    return true;
  }
  return false;
}

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {
  return pKeyData && compareVal(pKeyData, &pWin->stateKey);
}

4531 4532 4533
SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
X
Xiaoyu Wang 已提交
4534 4535
  int32_t           size = taosArrayGetSize(pWinInfos);
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index + 1;
      return pWin;
    }
  }
  *pIndex = 0;
  return NULL;
}

4556 4557
SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
                                 SColumn* pCol, int32_t* pIndex) {
4558 4559
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
5
54liuyao 已提交
4560 4561 4562 4563 4564
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    *pIndex = 0;
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
X
Xiaoyu Wang 已提交
4565
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
  *pIndex = index + 1;
  return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol);
}

H
Haojun Liao 已提交
4599 4600 4601
int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, uint64_t groupId,
                              SColumnInfoData* pKeyCol, int32_t rows, int32_t start, bool* allEqual,
                              SHashObj* pSeDeleted) {
5
54liuyao 已提交
4602 4603 4604 4605 4606
  *allEqual = true;
  SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex);
  for (int32_t i = start; i < rows; ++i) {
    char* pKeyData = colDataGetData(pKeyCol, i);
    if (!isTsInWindow(pWinInfo, pTs[i])) {
X
Xiaoyu Wang 已提交
4607
      if (isEqualStateKey(pWinInfo, pKeyData)) {
5
54liuyao 已提交
4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620
        int32_t size = taosArrayGetSize(pWinInfos);
        if (winIndex + 1 < size) {
          SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1);
          // ts belongs to the next window
          if (pTs[i] >= pNextWin->winInfo.win.skey) {
            return i - start;
          }
        }
      } else {
        return i - start;
      }
    }
    if (pWinInfo->winInfo.win.skey > pTs[i]) {
H
Haojun Liao 已提交
4621 4622
      if (pSeDeleted && pWinInfo->winInfo.isOutput) {
        SWinKey res = {.ts = pWinInfo->winInfo.win.skey, .groupId = groupId};
4623
        taosHashPut(pSeDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635
        pWinInfo->winInfo.isOutput = false;
      }
      pWinInfo->winInfo.win.skey = pTs[i];
    }
    pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]);
    if (!isEqualStateKey(pWinInfo, pKeyData)) {
      *allEqual = false;
    }
  }
  return rows - start;
}

H
Haojun Liao 已提交
4636 4637 4638 4639
static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SHashObj* pSeUpdated,
                                SHashObj* pSeDeleted) {
  SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupColInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
X
Xiaoyu Wang 已提交
4640 4641 4642
  TSKEY*           tsCol = (TSKEY*)pTsColInfo->pData;
  bool             allEqual = false;
  int32_t          step = 1;
H
Haojun Liao 已提交
4643
  uint64_t*        gpCol = (uint64_t*)pGroupColInfo->pData;
5
54liuyao 已提交
4644
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
4645
    int32_t           winIndex = 0;
H
Haojun Liao 已提交
4646
    SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], gpCol[i], &winIndex);
5
54liuyao 已提交
4647 4648 4649
    if (!pCurWin) {
      continue;
    }
H
Haojun Liao 已提交
4650
    updateSessionWindowInfo(&pCurWin->winInfo, tsCol, NULL, 0, pBlock->info.rows, i, 0, NULL);
5
54liuyao 已提交
4651
    taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
5
54liuyao 已提交
4652
    deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
5
54liuyao 已提交
4653 4654 4655
  }
}

X
Xiaoyu Wang 已提交
4656 4657 4658
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated,
                                 SHashObj* pStDeleted) {
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
4659
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4660
  bool                         masterScan = true;
4661
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
X
Xiaoyu Wang 已提交
4662 4663 4664 4665 4666 4667 4668
  int64_t                      groupId = pSDataBlock->info.groupId;
  int64_t                      code = TSDB_CODE_SUCCESS;
  int32_t                      step = 1;
  bool                         ascScan = true;
  TSKEY*                       tsCols = NULL;
  SResultRow*                  pResult = NULL;
  int32_t                      winRows = 0;
5
54liuyao 已提交
4669
  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
4670 4671
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
4672
  } else {
X
Xiaoyu Wang 已提交
4673
    return;
5
54liuyao 已提交
4674
  }
L
Liu Jicong 已提交
4675

5
54liuyao 已提交
4676
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
4677
  blockDataEnsureCapacity(pAggSup->pScanBlock, pSDataBlock->info.rows);
L
Liu Jicong 已提交
4678
  SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
X
Xiaoyu Wang 已提交
4679
  for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
5
54liuyao 已提交
4680
    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
4681 4682 4683
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
4684 4685 4686
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
    bool              allEqual = true;
H
Haojun Liao 已提交
4687 4688 4689
    SStateWindowInfo* pCurWin = getStateWindow(pAggSup, tsCols[i], groupId, pKeyData, &pInfo->stateCol, &winIndex);
    winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, groupId, pKeyColInfo, pSDataBlock->info.rows,
                                    i, &allEqual, pStDeleted);
5
54liuyao 已提交
4690
    if (!allEqual) {
4691
      uint64_t uid = 0;
L
Liu Jicong 已提交
4692
      appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, &uid, &groupId);
5
54liuyao 已提交
4693
      taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
5
54liuyao 已提交
4694
      deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
5
54liuyao 已提交
4695 4696
      continue;
    }
4697
    code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
4698
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
4699
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4700 4701 4702
    }
    pCurWin->winInfo.isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
H
Haojun Liao 已提交
4703 4704
      SWinKey value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId};
      code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
5
54liuyao 已提交
4705
      if (code != TSDB_CODE_SUCCESS) {
4706
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717
      }
      pCurWin->winInfo.isOutput = true;
    }
  }
}

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

4718
  SExprSupp*                   pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4719
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4720
  SOptrBasicInfo*              pBInfo = &pInfo->binfo;
L
Liu Jicong 已提交
4721
  int64_t                      maxTs = INT64_MIN;
5
54liuyao 已提交
4722 4723 4724
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4725
      printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4726 4727
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
4728
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
4729
    if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
5
54liuyao 已提交
4730 4731
      doSetOperatorCompleted(pOperator);
    }
5
54liuyao 已提交
4732
    printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4733 4734 4735
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
4736 4737
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
4738
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
4739
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
4740 4741 4742 4743 4744
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4745
    printDataBlock(pBlock, "single state recv");
4746

5
54liuyao 已提交
4747
    if (pBlock->info.type == STREAM_CLEAR) {
H
Haojun Liao 已提交
4748
      doClearStateWindows(&pInfo->streamAggSup, pBlock, pSeUpdated, pInfo->pSeDeleted);
5
54liuyao 已提交
4749
      continue;
4750 4751
    } else if (pBlock->info.type == STREAM_DELETE_DATA) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
5
54liuyao 已提交
4752
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo);
4753
      copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
4754
      removeSessionResults(pSeUpdated, pWins);
4755 4756
      taosArrayDestroy(pWins);
      continue;
4757
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4758
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState);
5
54liuyao 已提交
4759
      continue;
5
54liuyao 已提交
4760
    }
4761

5
54liuyao 已提交
4762 4763 4764 4765
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4766
    // the pDataBlock are always the same one, no need to call this again
4767
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4768
    doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
4769
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
4770
  }
4771
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4772 4773
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
X
Xiaoyu Wang 已提交
4774

4775
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState,
5
54liuyao 已提交
4776 4777
                     pInfo->ignoreExpiredData, destroyStateWinInfo);
  // closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, destroyStateWinInfo);
5
54liuyao 已提交
4778
  copyUpdateResult(pSeUpdated, pUpdated);
5
54liuyao 已提交
4779 4780
  taosHashCleanup(pSeUpdated);

4781 4782
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4783 4784 4785 4786
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4787
    printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4788 4789
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
4790
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
4791
  printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4792 4793 4794
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

4795 4796 4797 4798
int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo));
}

X
Xiaoyu Wang 已提交
4799 4800 4801 4802 4803 4804
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
  SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
4805
  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
4806

X
Xiaoyu Wang 已提交
4807 4808
  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
4809 4810 4811 4812
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

4813 4814
  SExprSupp* pSup = &pOperator->exprSupp;

X
Xiaoyu Wang 已提交
4815
  int32_t    numOfCols = 0;
5
54liuyao 已提交
4816 4817 4818
  SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
4819
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
4820 4821 4822 4823 4824 4825 4826 4827 4828
  if (pStateNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

4829
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
X
Xiaoyu Wang 已提交
4830 4831
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
5
54liuyao 已提交
4832 4833
      .calTrigger = pStateNode->window.triggerType,
      .maxTs = INT64_MIN,
X
Xiaoyu Wang 已提交
4834
  };
5
54liuyao 已提交
4835
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
4836

4837
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
4838 4839 4840 4841
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

4842
  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
4843 4844 4845 4846 4847 4848 4849 4850 4851
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }

4852
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
4853 4854 4855 4856 4857
  pInfo->primaryTsIndex = tsSlotId;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
H
Haojun Liao 已提交
4858
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
5
54liuyao 已提交
4859
  pInfo->pChildren = NULL;
5
54liuyao 已提交
4860
  pInfo->ignoreExpiredData = pStateNode->window.igExpired;
5
54liuyao 已提交
4861 4862

  pOperator->name = "StreamStateAggOperator";
4863
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
5
54liuyao 已提交
4864 4865 4866 4867
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
4868 4869
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL,
                                         destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
L
Liu Jicong 已提交
4870 4871
  initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType,
                 pInfo->primaryTsIndex);
5
54liuyao 已提交
4872 4873 4874 4875 4876 4877 4878
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
  return pOperator;

_error:
4879
  destroyStreamStateOperatorInfo(pInfo);
5
54liuyao 已提交
4880 4881 4882 4883
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
4884

4885
void destroyMergeAlignedIntervalOperatorInfo(void* param) {
4886
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param;
4887
  destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo);
D
dapan1121 已提交
4888
  taosMemoryFreeClear(param);
4889 4890
}

4891 4892
static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId,
                                                SSDataBlock* pResultBlock, TSKEY wstartTs) {
4893
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
4894

4895 4896 4897
  SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*            pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*                pSup = &pOperatorInfo->exprSupp;
4898

4899
  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId);
L
Liu Jicong 已提交
4900 4901
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(
      iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
4902
  ASSERT(p1 != NULL);
4903

4904 4905
  finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pSup->pCtx, pSup->pExprInfo, pSup->numOfExprs,
                                       pSup->rowEntryInfoOffset, pResultBlock, pTaskInfo);
4906 4907
  tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
  ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0);
4908

4909
  return TSDB_CODE_SUCCESS;
4910 4911
}

4912 4913
static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
                                          SSDataBlock* pBlock, int32_t scanFlag, SSDataBlock* pResultBlock) {
4914
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
D
dapan1121 已提交
4915
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
4916 4917

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
4918
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
4919 4920

  int32_t     startPos = 0;
4921
  int32_t     numOfOutput = pSup->numOfExprs;
4922
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
4923 4924 4925
  uint64_t    tableGroupId = pBlock->info.groupId;
  SResultRow* pResult = NULL;

4926 4927
  TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols);

4928 4929
  // there is an result exists
  if (miaInfo->curTs != INT64_MIN) {
4930
    ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1);
4931 4932

    if (ts != miaInfo->curTs) {
4933
      outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs);
4934
      miaInfo->curTs = ts;
4935
    }
4936 4937
  } else {
    miaInfo->curTs = ts;
4938
    ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0);
4939 4940 4941
  }

  STimeWindow win = {0};
4942
  win.skey = miaInfo->curTs;
4943 4944
  win.ekey =
      taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1;
4945

4946
  // TODO: remove the hash table (groupid + winkey => result row position)
4947 4948
  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
4949
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
4950
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
4951 4952
  }

4953 4954
  int32_t currPos = startPos;

4955
  STimeWindow currWin = win;
4956
  while (++currPos < pBlock->info.rows) {
4957
    if (tsCols[currPos] == miaInfo->curTs) {
4958
      continue;
4959 4960 4961
    }

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
4962 4963
    doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
                     pBlock->info.rows, numOfOutput);
4964

4965 4966
    outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs);
    miaInfo->curTs = tsCols[currPos];
4967

4968
    currWin.skey = miaInfo->curTs;
4969
    currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit,
4970 4971
                               iaInfo->interval.precision) -
                   1;
4972 4973 4974 4975 4976

    startPos = currPos;
    ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
4977
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
4978
    }
4979 4980

    miaInfo->curTs = currWin.skey;
4981
  }
4982

4983
  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
4984 4985
  doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
                   pBlock->info.rows, numOfOutput);
4986 4987
}

4988
static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
S
shenglian zhou 已提交
4989
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
4990

4991
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperator->info;
D
dapan1121 已提交
4992
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
4993

4994
  SExprSupp*   pSup = &pOperator->exprSupp;
4995
  SSDataBlock* pRes = iaInfo->binfo.pRes;
4996

4997 4998
  SOperatorInfo* downstream = pOperator->pDownstream[0];
  int32_t        scanFlag = MAIN_SCAN;
4999

5000 5001 5002 5003 5004 5005 5006
  while (1) {
    SSDataBlock* pBlock = NULL;
    if (miaInfo->prefetchedBlock == NULL) {
      pBlock = downstream->fpSet.getNextFn(downstream);
    } else {
      pBlock = miaInfo->prefetchedBlock;
      miaInfo->prefetchedBlock = NULL;
5007

5008 5009
      miaInfo->groupId = pBlock->info.groupId;
    }
5010

5011 5012 5013
    if (pBlock == NULL) {
      // close last unfinalized time window
      if (miaInfo->curTs != INT64_MIN) {
5014
        ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1);
5015 5016
        outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs);
        miaInfo->curTs = INT64_MIN;
5017
      }
5018

5019 5020
      doSetOperatorCompleted(pOperator);
      break;
5021
    }
5022

5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033
    if (!miaInfo->hasGroupId) {
      miaInfo->hasGroupId = true;
      miaInfo->groupId = pBlock->info.groupId;
    } else if (miaInfo->groupId != pBlock->info.groupId) {
      // if there are unclosed time window, close it firstly.
      ASSERT(miaInfo->curTs != INT64_MIN);
      outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs);
      miaInfo->prefetchedBlock = pBlock;
      miaInfo->curTs = INT64_MIN;
      break;
    }
5034

5035 5036 5037 5038 5039 5040 5041 5042
    getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag);
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true);
    doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);

    doFilter(miaInfo->pCondition, pRes, NULL);
    if (pRes->info.rows >= pOperator->resultInfo.capacity) {
      break;
    }
5043 5044
  }

5045
  pRes->info.groupId = miaInfo->groupId;
5046
  miaInfo->hasGroupId = false;
5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061
}

static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info;
  SIntervalAggOperatorInfo*             iaInfo = pMiaInfo->intervalAggOperatorInfo;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);

  if (iaInfo->binfo.mergeResultBlock) {
dengyihao's avatar
dengyihao 已提交
5062
    while (1) {
5063
      if (pOperator->status == OP_EXEC_DONE) {
5064 5065
        break;
      }
5066

5067
      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
5068 5069 5070
        break;
      }

5071 5072 5073 5074
      doMergeAlignedIntervalAgg(pOperator);
    }
  } else {
    doMergeAlignedIntervalAgg(pOperator);
5075 5076 5077 5078 5079 5080 5081
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

5082
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode,
5083
                                                      SExecTaskInfo* pTaskInfo) {
5084
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo));
5085
  SOperatorInfo*                        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5086 5087 5088 5089
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

D
dapan1121 已提交
5090 5091 5092 5093 5094
  miaInfo->intervalAggOperatorInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
  if (miaInfo->intervalAggOperatorInfo == NULL) {
    goto _error;
  }

5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105
  int32_t      num = 0;
  SExprInfo*   pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pNode->window.node.pOutputDataBlockDesc);

  SInterval interval = {.interval = pNode->interval,
                        .sliding = pNode->sliding,
                        .intervalUnit = pNode->intervalUnit,
                        .slidingUnit = pNode->slidingUnit,
                        .offset = pNode->offset,
                        .precision = ((SColumnNode*)pNode->window.pTspk)->node.resType.precision};

D
dapan1121 已提交
5106
  SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
5107
  SExprSupp*                pSup = &pOperator->exprSupp;
5108

5109
  miaInfo->pCondition = pNode->window.node.pConditions;
L
Liu Jicong 已提交
5110 5111 5112 5113 5114
  miaInfo->curTs = INT64_MIN;
  iaInfo->win = pTaskInfo->window;
  iaInfo->inputOrder = TSDB_ORDER_ASC;
  iaInfo->interval = interval;
  iaInfo->execModel = pTaskInfo->execModel;
5115 5116
  iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId;
  iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock;
5117 5118

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
5119
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5120

5121
  int32_t code = initAggInfo(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
5122 5123 5124
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
5125

5126
  initBasicInfo(&iaInfo->binfo, pResBlock);
5127
  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
5128

5129
  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, iaInfo);
5130
  if (iaInfo->timeWindowInterpo) {
5131
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
5132 5133
  }

5134
  initResultRowInfo(&iaInfo->binfo.resultRowInfo);
5135
  blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity);
5136

L
Liu Jicong 已提交
5137
  pOperator->name = "TimeMergeAlignedIntervalAggOperator";
5138
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL;
L
Liu Jicong 已提交
5139 5140 5141 5142
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = miaInfo;
5143

5144
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL,
5145
                                         destroyMergeAlignedIntervalOperatorInfo, NULL, NULL, NULL);
5146 5147 5148 5149 5150 5151 5152 5153 5154

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5155
  destroyMergeAlignedIntervalOperatorInfo(miaInfo);
5156 5157 5158 5159
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5160 5161 5162 5163 5164

//=====================================================================================================================
// merge interval operator
typedef struct SMergeIntervalAggOperatorInfo {
  SIntervalAggOperatorInfo intervalAggOperatorInfo;
L
Liu Jicong 已提交
5165 5166 5167 5168 5169 5170
  SList*                   groupIntervals;
  SListIter                groupIntervalsIter;
  bool                     hasGroupId;
  uint64_t                 groupId;
  SSDataBlock*             prefetchedBlock;
  bool                     inputBlocksFinished;
5171 5172
} SMergeIntervalAggOperatorInfo;

S
slzhou 已提交
5173
typedef struct SGroupTimeWindow {
L
Liu Jicong 已提交
5174
  uint64_t    groupId;
S
slzhou 已提交
5175 5176 5177
  STimeWindow window;
} SGroupTimeWindow;

5178
void destroyMergeIntervalOperatorInfo(void* param) {
5179
  SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
S
slzhou 已提交
5180
  tdListFree(miaInfo->groupIntervals);
5181
  destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo);
5182

D
dapan1121 已提交
5183
  taosMemoryFreeClear(param);
5184 5185
}

L
Liu Jicong 已提交
5186 5187
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win,
                                    SSDataBlock* pResultBlock) {
5188 5189 5190
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
5191
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5192 5193 5194
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId);
L
Liu Jicong 已提交
5195 5196
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(
      iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
5197 5198 5199
  ASSERT(p1 != NULL);
  finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo,
                                       pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo);
5200
  tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
5201 5202 5203
  return TSDB_CODE_SUCCESS;
}

5204 5205 5206 5207 5208
static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock,
                                        STimeWindow* newWin) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
5209
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5210 5211
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

S
slzhou 已提交
5212 5213
  SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin};
  tdListAppend(miaInfo->groupIntervals, &groupTimeWindow);
5214

S
slzhou 已提交
5215 5216 5217 5218 5219
  SListIter iter = {0};
  tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD);
  SListNode* listNode = NULL;
  while ((listNode = tdListNext(&iter)) != NULL) {
    SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
L
Liu Jicong 已提交
5220
    if (prevGrpWin->groupId != tableGroupId) {
S
slzhou 已提交
5221 5222 5223
      continue;
    }
    STimeWindow* prevWin = &prevGrpWin->window;
H
Haojun Liao 已提交
5224
    if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) {
S
slzhou 已提交
5225 5226 5227
      finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
      tdListPopNode(miaInfo->groupIntervals, listNode);
    }
5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244
  }

  return 0;
}

static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                                   int32_t scanFlag, SSDataBlock* pResultBlock) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*     pExprSup = &pOperatorInfo->exprSupp;

  int32_t     startPos = 0;
  int32_t     numOfOutput = pExprSup->numOfExprs;
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
5245
  bool        ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5246 5247 5248
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

5249 5250
  STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval,
                                        iaInfo->inputOrder);
5251 5252 5253 5254 5255

  int32_t ret =
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                             numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
5256
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5257 5258 5259 5260
  }

  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
5261
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5262 5263 5264 5265
  ASSERT(forwardRows > 0);

  // prev time window not interpolation yet.
  if (iaInfo->timeWindowInterpo) {
5266
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
5267 5268 5269 5270 5271 5272
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);

    // restore current time window
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                                 numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS) {
5273
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5274 5275 5276 5277 5278 5279 5280
    }

    // window start key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup);
  }

  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true);
5281 5282
  doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
                   pBlock->info.rows, numOfOutput);
5283 5284 5285 5286 5287 5288 5289 5290
  doCloseWindow(pResultRowInfo, iaInfo, pResult);

  // output previous interval results after this interval (&win) is closed
  outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win);

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = forwardRows - 1 + startPos;
5291 5292
    startPos =
        getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->inputOrder);
5293 5294 5295 5296 5297 5298 5299 5300 5301
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
    int32_t code =
        setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                               pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
5302
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5303 5304 5305 5306
    }

    ekey = ascScan ? nextWin.ekey : nextWin.skey;
    forwardRows =
5307
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5308 5309 5310 5311 5312

    // window start(end) key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup);

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true);
5313 5314
    doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pBlock->info.rows, numOfOutput);
5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354
    doCloseWindow(pResultRowInfo, iaInfo, pResult);

    // output previous interval results after this interval (&nextWin) is closed
    outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin);
  }

  if (iaInfo->timeWindowInterpo) {
    saveDataBlockLastRow(iaInfo->pPrevValues, pBlock, iaInfo->pInterpCols);
  }
}

static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExprSupp*                     pExpSupp = &pOperator->exprSupp;

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);

  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
        miaInfo->groupId = pBlock->info.groupId;
        miaInfo->prefetchedBlock = NULL;
      }

      if (pBlock == NULL) {
S
slzhou 已提交
5355
        tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD);
5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367
        miaInfo->inputBlocksFinished = true;
        break;
      }

      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
        miaInfo->groupId = pBlock->info.groupId;
      } else if (miaInfo->groupId != pBlock->info.groupId) {
        miaInfo->prefetchedBlock = pBlock;
        break;
      }

5368 5369
      getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag);
      setInputDataBlock(pOperator, pExpSupp->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true);
5370 5371 5372 5373 5374 5375 5376 5377
      doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);

      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
        break;
      }
    }

    pRes->info.groupId = miaInfo->groupId;
5378 5379 5380
  }

  if (miaInfo->inputBlocksFinished) {
S
slzhou 已提交
5381
    SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter);
5382

S
slzhou 已提交
5383 5384 5385 5386
    if (listNode != NULL) {
      SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data);
      finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
      pRes->info.groupId = grpWin->groupId;
5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398
    }
  }

  if (pRes->info.rows == 0) {
    doSetOperatorCompleted(pOperator);
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

5399 5400 5401
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode,
                                               SExecTaskInfo* pTaskInfo) {
  SMergeIntervalAggOperatorInfo* pMergeIntervalInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
5402
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5403
  if (pMergeIntervalInfo == NULL || pOperator == NULL) {
5404 5405 5406
    goto _error;
  }

5407 5408 5409 5410 5411 5412 5413 5414 5415 5416
  int32_t      num = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pIntervalPhyNode->window.node.pOutputDataBlockDesc);

  SInterval interval = {.interval = pIntervalPhyNode->interval,
                        .sliding = pIntervalPhyNode->sliding,
                        .intervalUnit = pIntervalPhyNode->intervalUnit,
                        .slidingUnit = pIntervalPhyNode->slidingUnit,
                        .offset = pIntervalPhyNode->offset,
                        .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
5417

5418
  pMergeIntervalInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow));
5419

5420
  SIntervalAggOperatorInfo* pIntervalInfo = &pMergeIntervalInfo->intervalAggOperatorInfo;
L
Liu Jicong 已提交
5421
  pIntervalInfo->win = pTaskInfo->window;
5422
  pIntervalInfo->inputOrder = TSDB_ORDER_ASC;
L
Liu Jicong 已提交
5423 5424
  pIntervalInfo->interval = interval;
  pIntervalInfo->execModel = pTaskInfo->execModel;
5425 5426
  pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock;
  pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
5427 5428 5429 5430

  SExprSupp* pExprSupp = &pOperator->exprSupp;

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
5431
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5432

5433
  int32_t code = initAggInfo(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
5434 5435 5436
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
5437

5438 5439
  initBasicInfo(&pIntervalInfo->binfo, pResBlock);
  initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win);
5440

5441 5442
  pIntervalInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, num, pIntervalInfo);
  if (pIntervalInfo->timeWindowInterpo) {
5443
    pIntervalInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
5444
    if (pIntervalInfo->binfo.resultRowInfo.openWindow == NULL) {
5445 5446 5447 5448
      goto _error;
    }
  }

5449
  initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo);
5450

L
Liu Jicong 已提交
5451
  pOperator->name = "TimeMergeIntervalAggOperator";
5452
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL;
L
Liu Jicong 已提交
5453 5454 5455 5456
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pMergeIntervalInfo;
5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL,
                                         destroyMergeIntervalOperatorInfo, NULL, NULL, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5469
  destroyMergeIntervalOperatorInfo(pMergeIntervalInfo);
5470 5471 5472
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
5473
}
5474

5475 5476
static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                                    int32_t scanFlag, SHashObj* pUpdatedMap) {
5477 5478 5479 5480 5481
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;

5482 5483
  int32_t          startPos = 0;
  int32_t          numOfOutput = pSup->numOfExprs;
5484
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
5485 5486 5487 5488 5489
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
  uint64_t         tableGroupId = pBlock->info.groupId;
  bool             ascScan = true;
  TSKEY            ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow*      pResult = NULL;
5490

5491 5492
  STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
  int32_t     ret = TSDB_CODE_SUCCESS;
5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552
  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
    }
  }

  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
  ASSERT(forwardRows > 0);

  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
  }

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = forwardRows - 1 + startPos;
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);
    if (startPos < 0) {
      break;
    }
    if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
      ekey = ascScan ? nextWin.ekey : nextWin.skey;
      forwardRows =
          getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
      continue;
    }

    // null data, failed to allocate more memory buffer
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
    }

    ekey = ascScan ? nextWin.ekey : nextWin.skey;
    forwardRows =
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
  }
}

5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629
static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
                                     SHashObj* pUpdatedMap) {
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;

  SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*  pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*      pSup = &pOperatorInfo->exprSupp;
  int32_t         numOfOutput = pSup->numOfExprs;
  int32_t         step = 1;
  TSKEY*          tsCols = NULL;
  SResultRow*     pResult = NULL;
  int32_t         forwardRows = 0;
  int32_t         aa = 4;

  ASSERT(pSDataBlock->pDataBlock != NULL);
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  tsCols = (int64_t*)pColDataInfo->pData;

  int32_t     startPos = 0;
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
  STimeWindow nextWin =
      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
  while (1) {
    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
    if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
      if (startPos < 0) {
        break;
      }
      continue;
    }

    int32_t code = setOutputBuf(&nextWin, &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset,
                                &pInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
                                           TSDB_ORDER_ASC);
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
    }
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pSDataBlock->info.rows, numOfOutput);
    SWinKey key = {
        .ts = nextWin.skey,
        .groupId = tableGroupId,
    };
    saveOutput(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize);
    releaseOutputBuf(pTaskInfo, &key, pResult);
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
    startPos =
        getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);
    if (startPos < 0) {
      break;
    }
  }
}

void doBuildResult(SOperatorInfo* pOperator, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  // set output datablock version
  pBlock->info.version = pTaskInfo->version;

  blockDataCleanup(pBlock);
  if (!hasRemainResults(pGroupResInfo)) {
    return;
  }

  // clear the existed group id
  pBlock->info.groupId = 0;
  buildDataBlockFromGroupRes(pTaskInfo, pBlock, &pOperator->exprSupp, pGroupResInfo);
}

5630 5631
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
5632 5633 5634
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
  int64_t                      maxTs = INT64_MIN;
  SExprSupp*                   pSup = &pOperator->exprSupp;
5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows > 0) {
      printDataBlock(pInfo->pDelRes, "single interval");
      return pInfo->pDelRes;
    }

    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
    if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
      pOperator->status = OP_EXEC_DONE;
      qDebug("===stream===single interval is done");
      freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
    }
    printDataBlock(pInfo->binfo.pRes, "single interval");
    return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];

  SArray*    pUpdated = taosArrayInit(4, POINTER_BYTES);  // SResKeyPos
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*  pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);

  SStreamState* pState = pTaskInfo->streamInfo.pState;

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
    printDataBlock(pBlock, "single interval recv");

    if (pBlock->info.type == STREAM_CLEAR) {
      doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock,
                     NULL);
      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
      continue;
    } else if (pBlock->info.type == STREAM_DELETE_DATA) {
      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval, pUpdatedMap);
      continue;
    } else if (pBlock->info.type == STREAM_GET_ALL) {
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
      continue;
    }

    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {
      // set input version
      pTaskInfo->version = pBlock->info.version;
    }

    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }

    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
    // caller. Note that all the time window are not close till now.
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
    if (pInfo->invertible) {
      setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type);
    }

    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
    doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap);
5705 5706
    // new disc buf
    // doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748
  }
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);

#if 0
  if (pState) {
    printf(">>>>>>>> stream read backend\n");
    SWinKey key = {
        .ts = 1,
        .groupId = 2,
    };
    char*   val = NULL;
    int32_t sz;
    if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) {
      ASSERT(0);
    }
    printf("stream read %s %d\n", val, sz);
    streamFreeVal(val);

    SStreamStateCur* pCur = streamStateGetCur(pState, &key);
    ASSERT(pCur);
    while (streamStateCurNext(pState, pCur) == 0) {
      SWinKey     key1;
      const void* val1;
      if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) {
        break;
      }
      printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz);
    }
    streamStateFreeCur(pCur);
  }
#endif

  pOperator->status = OP_RES_TO_RETURN;
  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap,
                      pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);

  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosArraySort(pUpdated, resultrowComparAsc);

5749
  // new disc buf
5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
  taosHashCleanup(pUpdatedMap);
  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows > 0) {
    printDataBlock(pInfo->pDelRes, "single interval");
    return pInfo->pDelRes;
  }

  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5762 5763
  // new disc buf
  // doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo);
5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784
  printDataBlock(pInfo->binfo.pRes, "single interval");
  return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}

void destroyStreamIntervalOperatorInfo(void* param) {
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;
  cleanupBasicInfo(&pInfo->binfo);
  cleanupAggSup(&pInfo->aggSup);
  pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages);

  pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins);
  pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);

  cleanupGroupResInfo(&pInfo->groupResInfo);
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  taosMemoryFreeClear(param);
}

SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
5785
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5786 5787 5788 5789 5790
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

5791 5792
  int32_t    numOfCols = 0;
  SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5793 5794
  ASSERT(numOfCols > 0);
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807
  SInterval    interval = {
         .interval = pIntervalPhyNode->interval,
         .sliding = pIntervalPhyNode->sliding,
         .intervalUnit = pIntervalPhyNode->intervalUnit,
         .slidingUnit = pIntervalPhyNode->slidingUnit,
         .offset = pIntervalPhyNode->offset,
         .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,
  };
  STimeWindowAggSupp twAggSupp = {
      .waterMark = pIntervalPhyNode->window.watermark,
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
  };
5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823
  ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY);
  pOperator->pTaskInfo = pTaskInfo;
  pInfo->interval = interval;
  pInfo->twAggSup = twAggSupp;
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
  pInfo->isFinal = false;

  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

5824
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
5825 5826
  initResultSizeInfo(&pOperator->resultInfo, 4096);
  SExprSupp* pSup = &pOperator->exprSupp;
5827 5828
  size_t     keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  int32_t    code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  initBasicInfo(&pInfo->binfo, pResBlock);
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
  pInfo->delIndex = 0;
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
  initResultRowInfo(&pInfo->binfo.resultRowInfo);

  pOperator->name = "StreamIntervalOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
5850 5851 5852
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamIntervalOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867

  initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
  destroyStreamIntervalOperatorInfo(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}