timewindowoperator.c 223.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15
#include "executorimpl.h"
X
Xiaoyu Wang 已提交
16
#include "function.h"
5
54liuyao 已提交
17
#include "functionMgt.h"
H
Haojun Liao 已提交
18
#include "tcommon.h"
5
54liuyao 已提交
19
#include "tcompare.h"
L
Liu Jicong 已提交
20
#include "tdatablock.h"
H
Haojun Liao 已提交
21
#include "tfill.h"
22
#include "ttime.h"
23 24 25 26 27 28

typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;

5
54liuyao 已提交
29 30
#define IS_FINAL_OP(op) ((op)->isFinal)

5
54liuyao 已提交
31 32
typedef struct SPullWindowInfo {
  STimeWindow window;
L
Liu Jicong 已提交
33
  uint64_t    groupId;
5
54liuyao 已提交
34 35
} SPullWindowInfo;

36 37
typedef struct SOpenWindowInfo {
  SResultRowPosition pos;
L
Liu Jicong 已提交
38
  uint64_t           groupId;
39 40
} SOpenWindowInfo;

5
54liuyao 已提交
41
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
5
54liuyao 已提交
42

43 44
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);

L
Liu Jicong 已提交
45 46
static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult,
                                              uint64_t groupId);
47 48
static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult);

H
Haojun Liao 已提交
49 50 51 52 53 54 55 56 57
///*
// * There are two cases to handle:
// *
// * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including
// * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey.
// * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be
// *    merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there
// *    is a previous result generated or not.
// */
58
// static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) {
H
Haojun Liao 已提交
59 60
//  // do nothing
//}
61

X
Xiaoyu Wang 已提交
62
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }
63 64 65

static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
66
                                      int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
67 68 69 70 71 72 73 74 75 76 77
                                      SExecTaskInfo* pTaskInfo) {
  SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                  masterscan, tableGroupId, pTaskInfo, true, pAggSup);

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // set time window for current result
  pResultRow->win = (*win);
78

79
  *pResult = pResultRow;
80
  setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
81

82 83 84 85 86 87 88 89 90 91 92 93 94
  return TSDB_CODE_SUCCESS;
}

static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) {
  int64_t* ts = (int64_t*)pColData->pData;
  int32_t  delta = includeEndpoint ? 1 : 0;

  int64_t duration = pWin->ekey - pWin->skey + delta;
  ts[2] = duration;            // set the duration
  ts[3] = pWin->skey;          // window start key
  ts[4] = pWin->ekey + delta;  // window end key
}

95
static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) {
96 97 98
  pRowSup->win.ekey = ts;
  pRowSup->prevTs = ts;
  pRowSup->numOfRows += 1;
99
  pRowSup->groupId = groupId;
100 101
}

dengyihao's avatar
dengyihao 已提交
102 103
static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex,
                                     uint64_t groupId) {
104 105 106
  pRowSup->startRowIndex = rowIndex;
  pRowSup->numOfRows = 0;
  pRowSup->win.skey = tsList[rowIndex];
107
  pRowSup->groupId = groupId;
108 109 110 111
}

static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
                                                   int16_t pos, int16_t order, int64_t* pData) {
112
  int32_t forwardRows = 0;
113 114 115 116

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
117
      forwardRows = end;
118 119

      if (pData[end + pos] == ekey) {
120
        forwardRows += 1;
121 122 123
      }
    }
  } else {
124
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
125
    if (end >= 0) {
126
      forwardRows = end;
127

128
      if (pData[end + pos] == ekey) {
129
        forwardRows += 1;
130 131
      }
    }
X
Xiaoyu Wang 已提交
132 133 134 135 136 137 138 139
    //    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
    //    if (end >= 0) {
    //      forwardRows = pos - end;
    //
    //      if (pData[end] == ekey) {
    //        forwardRows += 1;
    //      }
    //    }
140 141
  }

142 143
  assert(forwardRows >= 0);
  return forwardRows;
144 145
}

5
54liuyao 已提交
146
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY*  keyList = (TSKEY*)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
163 164 165 166 167 168 169 170 171 172 173
      if (key >= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key < keyList[lastPos]) {
        lastPos += 1;
        if (lastPos >= num) {
          return -1;
        } else {
          return lastPos;
        }
      }
174 175 176 177 178 179

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        firstPos = midPos + 1;
180 181
      } else if (key > keyList[midPos]) {
        lastPos = midPos - 1;
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

X
Xiaoyu Wang 已提交
217 218
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
                                 __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num = -1;
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);

  if (order == TSDB_ORDER_ASC) {
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
      if (item != NULL) {
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
240
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
241 242
      }
    } else {
243
      num = pDataBlockInfo->rows - startPos;
244
      if (item != NULL) {
245
        item->lastKey = pDataBlockInfo->window.ekey + step;
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
      }
    }
  }

  assert(num >= 0);
  return num;
}

static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
    return;
  }

  int64_t key = tw->skey, interval = pInterval->interval;
  // convert key to second
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;

  if (pInterval->intervalUnit == 'y') {
    interval *= 12;
  }

  struct tm tm;
  time_t    t = (time_t)key;
  taosLocalTime(&t, &tm);

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
277
  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
278 279 280 281

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
282
  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
283 284 285 286

  tw->ekey -= 1;
}

287 288
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs,
                               int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
289
  SqlFunctionCtx* pCtx = pSup->pCtx;
290

291
  int32_t index = 1;
292
  for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
H
Haojun Liao 已提交
293
    if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
294 295 296 297
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

X
Xiaoyu Wang 已提交
298
    SFunctParam*     pParam = &pCtx[k].param[0];
299 300
    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId);

301
    ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey);
302

303
    double v1 = 0, v2 = 0, v = 0;
304
    if (prevRowIndex == -1) {
305
      SGroupKeys* p = taosArrayGet(pPrevValues, index);
306
      GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData);
307
    } else {
308
      GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex));
309 310
    }

311
    GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
312

313
#if 0
314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            //            pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
333 334
#endif

X
Xiaoyu Wang 已提交
335 336 337
    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
    SPoint point = (SPoint){.key = windowKey, .val = &v};
338

X
Xiaoyu Wang 已提交
339
    taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
340

X
Xiaoyu Wang 已提交
341 342 343 344 345 346
    if (type == RESULT_ROW_START_INTERP) {
      pCtx[k].start.key = point.key;
      pCtx[k].start.val = v;
    } else {
      pCtx[k].end.key = point.key;
      pCtx[k].end.val = v;
347
    }
X
Xiaoyu Wang 已提交
348 349 350

    index += 1;
  }
351
#if 0
352
  }
353
#endif
354 355 356 357 358 359 360 361 362 363 364 365 366 367
}

static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

368 369
static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock,
                                              const TSKEY* tsCols, STimeWindow* win, SExprSupp* pSup) {
370
  bool ascQuery = (pInfo->inputOrder == TSDB_ORDER_ASC);
371

372
  TSKEY curTs = tsCols[pos];
373 374

  SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
X
Xiaoyu Wang 已提交
375
  TSKEY       lastTs = *(int64_t*)pTsKey->pData;
376 377 378 379 380

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery ? win->skey : win->ekey;
  if (key == curTs) {
381
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
382 383 384
    return true;
  }

385 386
  // it is the first time window, no need to do interpolation
  if (pTsKey->isNull && pos == 0) {
387
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
388 389
  } else {
    TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]);
390 391
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key,
                              RESULT_ROW_START_INTERP, pSup);
392 393 394 395 396
  }

  return true;
}

397 398 399
static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex,
                                            SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey,
                                            STimeWindow* win) {
400
  int32_t order = pInfo->inputOrder;
401 402

  TSKEY actualEndKey = tsCols[endRowIndex];
403
  TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey;
404 405

  // not ended in current data block, do not invoke interpolation
406
  if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) {
407
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
408 409 410
    return false;
  }

411
  // there is actual end point of current time window, no interpolation needs
412
  if (key == actualEndKey) {
413
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
414 415 416
    return true;
  }

417
  int32_t nextRowIndex = endRowIndex + 1;
418 419 420
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
421 422
  doTimeWindowInterpolation(pInfo->pPrevValues, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key,
                            RESULT_ROW_END_INTERP, pSup);
423 424 425
  return true;
}

5
54liuyao 已提交
426
bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo) {
427 428
  if (pInterval->interval != pInterval->sliding &&
      (pWin->ekey < pBlockInfo->calWin.skey || pWin->skey > pBlockInfo->calWin.ekey)) {
5
54liuyao 已提交
429 430 431 432 433
    return false;
  }
  return true;
}

434
static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
5
54liuyao 已提交
435
                                      TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
X
Xiaoyu Wang 已提交
436
  bool ascQuery = (order == TSDB_ORDER_ASC);
437 438 439 440 441 442 443 444 445 446

  int32_t precision = pInterval->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
    return -1;
  }

5
54liuyao 已提交
447 448 449 450
  if (!inSlidingWindow(pInterval, pNext, pDataBlockInfo) && order == TSDB_ORDER_ASC) {
    return -1;
  }

451
  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
452 453 454 455
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
456
    startPos = prevPosition + 1;
457
  } else {
458
    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
459 460
      startPos = 0;
    } else {
461
      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
    }
  }

  /* interp query with fill should not skip time window */
  //  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
  //    return startPos;
  //  }

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
    if (ascQuery) {
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
      }
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
      }
    }
  }

  return startPos;
}

505 506
static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
  ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp == true;
  }
}

static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp = true;
  }
}

523 524
static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult,
                                        STimeWindow* win, int32_t startPos, int32_t forwardRows, SExprSupp* pSup) {
525
  if (!pInfo->timeWindowInterpo) {
526 527 528
    return;
  }

529
  ASSERT(pBlock != NULL);
530 531 532 533 534
  if (pBlock->pDataBlock == NULL) {
    //    tscError("pBlock->pDataBlock == NULL");
    return;
  }

535
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
536 537

  TSKEY* tsCols = (TSKEY*)(pColInfo->pData);
538
  bool   done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
539
  if (!done) {  // it is not interpolated, now start to generated the interpolated value
540
    bool interp = setTimeWindowInterpolationStartTs(pInfo, startPos, pBlock, tsCols, win, pSup);
541 542 543 544
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
545
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
546 547 548 549 550 551 552 553
  }

  // point interpolation does not require the end key time window interpolation.
  //  if (pointInterpQuery) {
  //    return;
  //  }

  // interpolation query does not generate the time window end interpolation
554
  done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
555
  if (!done) {
556
    int32_t endRowIndex = startPos + forwardRows - 1;
557

558
    TSKEY endKey = (pInfo->inputOrder == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
559
    bool  interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
560 561 562 563
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
564
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
565 566 567
  }
}

568 569
static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) {
  if (pBlock->pDataBlock == NULL) {
570 571 572
    return;
  }

573 574 575 576 577 578 579
  size_t num = taosArrayGetSize(pPrevKeys);
  for (int32_t k = 0; k < num; ++k) {
    SColumn* pc = taosArrayGet(pCols, k);

    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId);

    SGroupKeys* pkey = taosArrayGet(pPrevKeys, k);
X
Xiaoyu Wang 已提交
580
    for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) {
581 582 583 584 585 586 587 588 589 590 591 592 593 594
      if (colDataIsNull_s(pColInfo, i)) {
        continue;
      }

      char* val = colDataGetData(pColInfo, i);
      if (IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, varDataTLen(val));
        ASSERT(varDataTLen(val) <= pkey->bytes);
      } else {
        memcpy(pkey->pData, val, pkey->bytes);
      }

      break;
    }
595 596 597
  }
}

598 599 600 601
static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo,
                                       SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) {
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

602
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
603
  SExprSupp*                pSup = &pOperatorInfo->exprSupp;
604

L
Liu Jicong 已提交
605 606
  int32_t startPos = 0;
  int32_t numOfOutput = pSup->numOfExprs;
607

608
  SResultRow* pResult = NULL;
609

610
  while (1) {
L
Liu Jicong 已提交
611 612 613
    SListNode*          pn = tdListGetHead(pResultRowInfo->openWindow);
    SOpenWindowInfo*    pOpenWin = (SOpenWindowInfo*)pn->data;
    uint64_t            groupId = pOpenWin->groupId;
614
    SResultRowPosition* p1 = &pOpenWin->pos;
615 616 617
    if (p->pageId == p1->pageId && p->offset == p1->offset) {
      break;
    }
618

619
    SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1, false);
620
    ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
621

622
    if (pr->closed) {
X
Xiaoyu Wang 已提交
623 624
      ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
             isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
625 626
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
627 628
      continue;
    }
629

630
    STimeWindow w = pr->win;
631 632
    int32_t     ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx,
                                             numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
633
    if (ret != TSDB_CODE_SUCCESS) {
634
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
635 636 637 638
    }

    ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP));

X
Xiaoyu Wang 已提交
639 640
    SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
    int64_t     prevTs = *(int64_t*)pTsKey->pData;
641 642 643 644
    if (groupId == pBlock->info.groupId) {
      doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey,
                                RESULT_ROW_END_INTERP, pSup);
    }
645 646

    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
647
    setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
648

649
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, true);
H
Haojun Liao 已提交
650 651
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, pBlock->info.rows,
                     numOfExprs);
652 653 654

    if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
      closeResultRow(pr);
655 656
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
X
Xiaoyu Wang 已提交
657
    } else {  // the remains are can not be closed yet.
658
      break;
659
    }
660
  }
661
}
662

5
54liuyao 已提交
663
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
5
54liuyao 已提交
664
  if (!pBlock || pBlock->info.rows == 0) {
5
54liuyao 已提交
665
    qDebug("===stream===printDataBlock: Block is Null or Empty");
5
54liuyao 已提交
666 667
    return;
  }
L
Liu Jicong 已提交
668
  char* pBuf = NULL;
5
54liuyao 已提交
669 670
  qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
  taosMemoryFree(pBuf);
5
54liuyao 已提交
671 672
}

5
54liuyao 已提交
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);

int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;
      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;
      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;
      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

      if (comparefn(pKey, keyList, lastPos) > 0) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

5
54liuyao 已提交
729
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
730

X
Xiaoyu Wang 已提交
731 732 733
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;
5
54liuyao 已提交
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (key >= getValuefn(keyList, lastPos)) return lastPos;
      if (key == getValuefn(keyList, firstPos)) return firstPos;
      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (key <= getValuefn(keyList, firstPos)) return firstPos;
      if (key == getValuefn(keyList, lastPos)) return lastPos;

      if (key > getValuefn(keyList, lastPos)) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
780 781
  }

5
54liuyao 已提交
782 783 784
  return midPos;
}

5
54liuyao 已提交
785
int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {
L
Liu Jicong 已提交
786
  SArray*          res = (SArray*)data;
5
54liuyao 已提交
787
  SPullWindowInfo* pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
788
  SPullWindowInfo* pData = (SPullWindowInfo*)pKey;
5
54liuyao 已提交
789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819
  if (pData->window.skey == pos->window.skey) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->window.skey > pos->window.skey) {
    return 1;
  }
  return -1;
}

static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {
  int32_t size = taosArrayGetSize(pPullWins);
  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);
  if (index == -1) {
    index = 0;
  } else {
    if (comparePullWinKey(pPullInfo, pPullWins, index) > 0) {
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
820 821 822
int32_t compareResKey(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
  SResKeyPos* pos = taosArrayGetP(res, index);
H
Haojun Liao 已提交
823
  SWinKey*    pData = (SWinKey*)pKey;
5
54liuyao 已提交
824 825 826 827 828 829 830 831 832 833 834 835 836
  if (pData->ts == *(int64_t*)pos->key) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->ts > *(int64_t*)pos->key) {
    return 1;
  }
  return -1;
}

837
static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SArray* pUpdated) {
5
54liuyao 已提交
838
  int32_t size = taosArrayGetSize(pUpdated);
H
Haojun Liao 已提交
839
  SWinKey data = {.ts = ts, .groupId = groupId};
5
54liuyao 已提交
840
  int32_t index = binarySearchCom(pUpdated, size, &data, TSDB_ORDER_DESC, compareResKey);
5
54liuyao 已提交
841 842 843
  if (index == -1) {
    index = 0;
  } else {
5
54liuyao 已提交
844
    if (compareResKey(&data, pUpdated, index) > 0) {
5
54liuyao 已提交
845 846 847 848 849
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
H
Haojun Liao 已提交
850

5
54liuyao 已提交
851 852 853 854 855
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
5
54liuyao 已提交
856 857
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
X
Xiaoyu Wang 已提交
858
  if (taosArrayInsert(pUpdated, index, &newPos) == NULL) {
5
54liuyao 已提交
859 860 861 862 863
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
864 865 866 867 868 869 870 871
static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SHashObj* pUpdatedMap) {
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
H
Haojun Liao 已提交
872 873
  SWinKey key = {.ts = ts, .groupId = groupId};
  if (taosHashPut(pUpdatedMap, &key, sizeof(SWinKey), &newPos, sizeof(void*)) != TSDB_CODE_SUCCESS) {
5
54liuyao 已提交
874 875 876
    taosMemoryFree(newPos);
  }
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
877 878
}

5
54liuyao 已提交
879
static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj* pUpdatedMap) {
dengyihao's avatar
dengyihao 已提交
880
  return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap);
5
54liuyao 已提交
881 882 883 884
}

static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
  return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated);
5
54liuyao 已提交
885 886
}

5
54liuyao 已提交
887
static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) {
5
54liuyao 已提交
888 889
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
H
Haojun Liao 已提交
890 891
    SWinKey* pW = taosArrayGet(pWins, i);
    taosHashRemove(pUpdatedMap, pW, sizeof(SWinKey));
5
54liuyao 已提交
892 893 894
  }
}

895
int64_t getWinReskey(void* data, int32_t index) {
896
  SArray*  res = (SArray*)data;
H
Haojun Liao 已提交
897
  SWinKey* pos = taosArrayGet(res, index);
898 899 900
  return pos->ts;
}

5
54liuyao 已提交
901 902
int32_t compareWinRes(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
903
  SWinKey*    pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
904
  SResKeyPos* pData = (SResKeyPos*)pKey;
5
54liuyao 已提交
905 906 907 908 909 910 911 912 913 914 915 916 917
  if (*(int64_t*)pData->key == pos->ts) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (*(int64_t*)pData->key > pos->ts) {
    return 1;
  }
  return -1;
}

5
54liuyao 已提交
918
static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) {
L
Liu Jicong 已提交
919 920
  int32_t delSize = taosArrayGetSize(pDelWins);
  if (taosHashGetSize(pUpdatedMap) == 0 || delSize == 0) {
5
54liuyao 已提交
921
    return;
dengyihao's avatar
dengyihao 已提交
922
  }
L
Liu Jicong 已提交
923
  void* pIte = NULL;
5
54liuyao 已提交
924
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
925
    SResKeyPos* pResKey = *(SResKeyPos**)pIte;
5
54liuyao 已提交
926 927
    int32_t     index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes);
    if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) {
928
      taosArrayRemove(pDelWins, index);
929
      delSize = taosArrayGetSize(pDelWins);
930 931 932 933
    }
  }
}

5
54liuyao 已提交
934 935 936 937 938
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) {
  ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
  return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark;
}

L
Liu Jicong 已提交
939
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); }
5
54liuyao 已提交
940

5
54liuyao 已提交
941
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
942
                            int32_t scanFlag) {
943
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
944

945
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
946
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
947

X
Xiaoyu Wang 已提交
948
  int32_t     startPos = 0;
949
  int32_t     numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
950 951
  int64_t*    tsCols = extractTsCol(pBlock, pInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
952
  bool        ascScan = (pInfo->inputOrder == TSDB_ORDER_ASC);
X
Xiaoyu Wang 已提交
953 954
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;
955

956 957
  STimeWindow win =
      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder);
958 959 960 961
  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
962
  }
X
Xiaoyu Wang 已提交
963 964
  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
965
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
966
  ASSERT(forwardRows > 0);
967 968

  // prev time window not interpolation yet.
969
  if (pInfo->timeWindowInterpo) {
970
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
971
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);
972 973

    // restore current time window
974 975
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
976
    if (ret != TSDB_CODE_SUCCESS) {
977
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
978 979
    }

980
    // window start key interpolation
981
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
982
  }
983

984 985 986
  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                    numOfOutput);
987 988

  doCloseWindow(pResultRowInfo, pInfo, pResult);
989 990 991

  STimeWindow nextWin = win;
  while (1) {
992
    int32_t prevEndPos = forwardRows - 1 + startPos;
993
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->inputOrder);
994 995 996 997
    if (startPos < 0) {
      break;
    }
    // null data, failed to allocate more memory buffer
X
Xiaoyu Wang 已提交
998
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
999
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1000
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
1001
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
1002 1003
    }

X
Xiaoyu Wang 已提交
1004
    ekey = ascScan ? nextWin.ekey : nextWin.skey;
1005
    forwardRows =
1006
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
1007
    // window start(end) key interpolation
1008
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
L
Liu Jicong 已提交
1009
    // TODO: add to open window? how to close the open windows after input blocks exhausted?
S
shenglian zhou 已提交
1010
#if 0
1011 1012 1013 1014
    if ((ascScan && ekey <= pBlock->info.window.ekey) ||
        (!ascScan && ekey >= pBlock->info.window.skey)) {
      // window start(end) key interpolation
      doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
1015
    } else if (pInfo->timeWindowInterpo) {
1016 1017
      addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
    }
S
shenglian zhou 已提交
1018
#endif
1019
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
H
Haojun Liao 已提交
1020 1021
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
1022
    doCloseWindow(pResultRowInfo, pInfo, pResult);
1023 1024 1025
  }

  if (pInfo->timeWindowInterpo) {
1026
    saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
1027
  }
1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
}

void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
  // current result is done in computing final results.
  if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
    closeResultRow(pResult);
    tdListPopHead(pResultRowInfo->openWindow);
  }
}

1038 1039 1040 1041 1042
SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, uint64_t groupId) {
  SOpenWindowInfo openWin = {0};
  openWin.pos.pageId = pResult->pageId;
  openWin.pos.offset = pResult->offset;
  openWin.groupId = groupId;
L
Liu Jicong 已提交
1043
  SListNode* pn = tdListGetTail(pResultRowInfo->openWindow);
1044
  if (pn == NULL) {
1045 1046
    tdListAppend(pResultRowInfo->openWindow, &openWin);
    return openWin.pos;
1047 1048
  }

L
Liu Jicong 已提交
1049
  SOpenWindowInfo* px = (SOpenWindowInfo*)pn->data;
1050 1051
  if (px->pos.pageId != openWin.pos.pageId || px->pos.offset != openWin.pos.offset || px->groupId != openWin.groupId) {
    tdListAppend(pResultRowInfo->openWindow, &openWin);
1052 1053
  }

1054
  return openWin.pos;
1055 1056 1057 1058
}

int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
  TSKEY* tsCols = NULL;
1059

1060 1061 1062 1063
  if (pBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;

1064 1065 1066 1067 1068 1069
    // no data in primary ts
    if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) {
      return NULL;
    }

    if (tsCols[0] != 0 && (pBlock->info.window.skey == 0 && pBlock->info.window.ekey == 0)) {
1070 1071 1072 1073 1074
      blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
    }
  }

  return tsCols;
1075 1076 1077 1078 1079 1080 1081
}

static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
  }

L
Liu Jicong 已提交
1082
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1083
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
1084
  SExprSupp*                pSup = &pOperator->exprSupp;
1085

1086 1087
  int32_t scanFlag = MAIN_SCAN;

X
Xiaoyu Wang 已提交
1088
  int64_t        st = taosGetTimestampUs();
1089 1090 1091
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1092
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1093 1094 1095 1096
    if (pBlock == NULL) {
      break;
    }

1097
    getTableScanInfo(pOperator, &pInfo->inputOrder, &scanFlag);
1098

1099
    if (pInfo->scalarSupp.pExprInfo != NULL) {
L
Liu Jicong 已提交
1100 1101
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
1102 1103
    }

1104
    // the pDataBlock are always the same one, no need to call this again
1105
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, scanFlag, true);
1106 1107
    blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);

1108
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag);
1109 1110
  }

1111
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->resultTsOrder);
1112
  OPTR_SET_OPENED(pOperator);
1113 1114

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1115 1116 1117
  return TSDB_CODE_SUCCESS;
}

1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
static bool compareVal(const char* v, const SStateKeys* pKey) {
  if (IS_VAR_DATA_TYPE(pKey->type)) {
    if (varDataLen(v) != varDataLen(pKey->pData)) {
      return false;
    } else {
      return strncmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0;
    }
  } else {
    return memcmp(pKey->pData, v, pKey->bytes) == 0;
  }
}

1130
static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
L
Liu Jicong 已提交
1131
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1132
  SExprSupp*     pSup = &pOperator->exprSupp;
1133

1134
  SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
1135 1136 1137
  int64_t          gid = pBlock->info.groupId;

  bool    masterScan = true;
1138
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1139 1140
  int16_t bytes = pStateColInfoData->info.bytes;

1141
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1142 1143 1144 1145 1146
  TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

1147
  struct SColumnDataAgg* pAgg = NULL;
1148
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
X
Xiaoyu Wang 已提交
1149
    pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
1150
    if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
1151 1152 1153 1154 1155
      continue;
    }

    char* val = colDataGetData(pStateColInfoData, j);

1156
    if (gid != pRowSup->groupId || !pInfo->hasKey) {
1157 1158 1159 1160 1161 1162 1163
      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }

1164 1165
      pInfo->hasKey = true;

1166 1167
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1168
    } else if (compareVal(val, &pInfo->stateKey)) {
1169
      doKeepTuple(pRowSup, tsList[j], gid);
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // a new state window started
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1180 1181
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1182
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1183
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1184 1185 1186
      }

      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1187 1188
      doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                       pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1189 1190

      // here we start a new session window
1191 1192
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1193 1194 1195 1196 1197 1198 1199

      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }
1200 1201 1202 1203 1204
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1205 1206
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1207
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1208
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1209 1210 1211
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
H
Haojun Liao 已提交
1212 1213
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
                   pBlock->info.rows, numOfOutput);
1214 1215
}

1216
static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
1217 1218 1219 1220 1221
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
1222

1223 1224
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  SExprSupp*     pSup = &pOperator->exprSupp;
1225

1226
  SOptrBasicInfo* pBInfo = &pInfo->binfo;
1227 1228

  if (pOperator->status == OP_RES_TO_RETURN) {
1229
    while (1) {
1230
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1231
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1232

1233
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }

      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1244
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1245 1246
  }

1247
  int32_t order = TSDB_ORDER_ASC;
1248
  int64_t st = taosGetTimestampUs();
1249 1250 1251

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  while (1) {
1252
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1253 1254 1255 1256
    if (pBlock == NULL) {
      break;
    }

1257
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1258 1259
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1260 1261 1262
    doStateWindowAggImpl(pOperator, pInfo, pBlock);
  }

X
Xiaoyu Wang 已提交
1263
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1264 1265
  pOperator->status = OP_RES_TO_RETURN;

1266
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1267
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1268
  while (1) {
1269
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1270
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1271

1272
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1273 1274 1275 1276
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
1277

1278 1279 1280 1281 1282
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1283
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1284 1285
}

1286
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
1287
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
1288
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1289 1290 1291 1292 1293 1294 1295 1296

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pBlock = pInfo->binfo.pRes;

  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
1297
    return pOperator->fpSet.getStreamResFn(pOperator);
1298 1299 1300 1301 1302 1303 1304
  } else {
    pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
    if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
      return NULL;
    }

    blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
1305 1306
    while (1) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1307
      doFilter(pInfo->pCondition, pBlock, NULL);
1308

1309
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1310 1311 1312 1313
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
1314

1315 1316 1317
      if (pBlock->info.rows > 0) {
        break;
      }
1318 1319
    }

1320 1321 1322
    size_t rows = pBlock->info.rows;
    pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1323
    return (rows == 0) ? NULL : pBlock;
1324 1325 1326 1327
  }
}

// todo merged with the build group result.
1328
static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
1329
                                  int32_t* rowEntryInfoOffset) {
1330 1331 1332 1333 1334 1335 1336
  size_t num = taosArrayGetSize(pUpdateList);

  for (int32_t i = 0; i < num; ++i) {
    SResKeyPos* pPos = taosArrayGetP(pUpdateList, i);

    SFilePage*  bufPage = getBufPage(pBuf, pPos->pos.pageId);
    SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
1337

1338
    for (int32_t j = 0; j < numOfOutput; ++j) {
1339
      SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, j, rowEntryInfoOffset);
1340 1341
      if (pRow->numOfRows < pEntry->numOfRes) {
        pRow->numOfRows = pEntry->numOfRes;
1342 1343 1344 1345 1346 1347
      }
    }

    releaseBufPage(pBuf, bufPage);
  }
}
5
54liuyao 已提交
1348
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
L
Liu Jicong 已提交
1349
  for (int i = 0; i < num; i++) {
5
54liuyao 已提交
1350 1351
    if (type == STREAM_INVERT) {
      fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
L
Liu Jicong 已提交
1352
    } else if (type == STREAM_NORMAL) {
5
54liuyao 已提交
1353 1354 1355 1356
      fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
    }
  }
}
5
54liuyao 已提交
1357

1358
void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) {
1359
  SResultRow*     pResult = getResultRowByPos(pResultBuf, p1, false);
1360
  SqlFunctionCtx* pCtx = pSup->pCtx;
5
54liuyao 已提交
1361
  for (int32_t i = 0; i < numOfOutput; ++i) {
1362
    pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
1363 1364 1365 1366 1367 1368 1369 1370 1371
    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
      continue;
    }
    pResInfo->initialized = false;
    if (pCtx[i].functionId != -1) {
      pCtx[i].fpSet.init(&pCtx[i], pResInfo);
    }
  }
5
54liuyao 已提交
1372 1373 1374
  SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pResultBuf, bufPage);
5
54liuyao 已提交
1375 1376
}

5
54liuyao 已提交
1377
bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
X
Xiaoyu Wang 已提交
1378
                   int32_t numOfOutput) {
1379
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId);
5
54liuyao 已提交
1380
  SResultRowPosition* p1 =
1381
      (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1382 1383
  if (!p1) {
    // window has been closed
5
54liuyao 已提交
1384
    return false;
1385
  }
1386
  doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput);
5
54liuyao 已提交
1387
  return true;
5
54liuyao 已提交
1388 1389
}

1390 1391 1392 1393
bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) {
  size_t bytes = sizeof(TSKEY);
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId);
  SResultRowPosition* p1 =
1394
      (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1395 1396 1397 1398
  if (!p1) {
    // window has been closed
    return false;
  }
5
54liuyao 已提交
1399
  // SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId);
1400
  // dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage);
1401
  tSimpleHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1402 1403 1404
  return true;
}

L
Liu Jicong 已提交
1405 1406
void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pDelWins, SInterval* pInterval,
                                   SHashObj* pUpdatedMap) {
1407
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
1408
  TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
1409
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
1410
  uint64_t*        groupIds = (uint64_t*)pGroupCol->pData;
1411 1412 1413
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
1414
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC);
1415
    doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
1416 1417 1418 1419 1420 1421
    SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]};
    if (pDelWins) {
      taosArrayPush(pDelWins, &winRes);
    }
    if (pUpdatedMap) {
      taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
1422 1423 1424 1425
    }
  }
}

1426 1427
static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput,
                           SSDataBlock* pBlock, SArray* pUpWins) {
1428 1429 1430 1431
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  TSKEY*           startTsCols = (TSKEY*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           endTsCols = (TSKEY*)pEndTsCol->pData;
1432 1433
  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*        pGpDatas = (uint64_t*)pGpCol->pData;
1434 1435 1436 1437 1438
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
    while (win.ekey <= endTsCols[i]) {
1439
      uint64_t winGpId = pGpDatas[i];
L
Liu Jicong 已提交
1440
      bool     res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput);
1441
      if (pUpWins && res) {
H
Haojun Liao 已提交
1442
        SWinKey winRes = {.ts = win.skey, .groupId = winGpId};
1443 1444 1445
        taosArrayPush(pUpWins, &winRes);
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
5
54liuyao 已提交
1446
    }
5
54liuyao 已提交
1447 1448
  }
}
1449

1450 1451 1452 1453 1454 1455
static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) {
  void*   pIte = NULL;
  size_t  keyLen = 0;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    void*    key = tSimpleHashGetKey(pIte, &keyLen);
5
54liuyao 已提交
1456 1457
    uint64_t groupId = *(uint64_t*)key;
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
1458
    TSKEY               ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1459
    SResultRowPosition* pPos = (SResultRowPosition*)pIte;
5
54liuyao 已提交
1460
    int32_t             code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
5
54liuyao 已提交
1461 1462 1463 1464 1465 1466 1467
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }
  return TSDB_CODE_SUCCESS;
}

1468
static int32_t closeIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
5
54liuyao 已提交
1469
                                   SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pRecyPages,
1470
                                   SDiskbasedBuf* pDiscBuf) {
1471
  qDebug("===stream===close interval window");
1472 1473 1474 1475 1476
  void*   pIte = NULL;
  size_t  keyLen = 0;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    void*    key = tSimpleHashGetKey(pIte, &keyLen);
X
Xiaoyu Wang 已提交
1477
    uint64_t groupId = *(uint64_t*)key;
5
54liuyao 已提交
1478
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
L
Liu Jicong 已提交
1479
    TSKEY       ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1480 1481 1482
    STimeWindow win;
    win.skey = ts;
    win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
H
Haojun Liao 已提交
1483
    SWinKey winRe = {
L
Liu Jicong 已提交
1484 1485
        .ts = win.skey,
        .groupId = groupId,
L
Liu Jicong 已提交
1486
    };
H
Haojun Liao 已提交
1487
    void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinKey));
5
54liuyao 已提交
1488
    if (isCloseWindow(&win, pSup)) {
5
54liuyao 已提交
1489
      if (chIds && pPullDataMap) {
L
Liu Jicong 已提交
1490
        SArray* chAy = *(SArray**)chIds;
5
54liuyao 已提交
1491
        int32_t size = taosArrayGetSize(chAy);
5
54liuyao 已提交
1492
        qDebug("===stream===window %" PRId64 " wait child size:%d", win.skey, size);
5
54liuyao 已提交
1493
        for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
1494
          qDebug("===stream===window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i));
5
54liuyao 已提交
1495 1496 1497
        }
        continue;
      } else if (pPullDataMap) {
5
54liuyao 已提交
1498
        qDebug("===stream===close window %" PRId64, win.skey);
5
54liuyao 已提交
1499
      }
5
54liuyao 已提交
1500 1501
      SResultRowPosition* pPos = (SResultRowPosition*)pIte;
      if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
1502
        int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, closeWins);
5
54liuyao 已提交
1503 1504 1505
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
1506 1507 1508
        ASSERT(pRecyPages != NULL);
        taosArrayPush(pRecyPages, &pPos->pageId);
      } else {
5
54liuyao 已提交
1509
        // SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId);
1510
        // dBufSetBufPageRecycled(pDiscBuf, bufPage);
5
54liuyao 已提交
1511
      }
5
54liuyao 已提交
1512 1513
      char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
      SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
1514
      tSimpleHashIterateRemove(pHashMap, keyBuf, keyLen, &pIte, &iter);
5
54liuyao 已提交
1515 1516 1517 1518 1519
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
1520 1521 1522
static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) {
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
L
Liu Jicong 已提交
1523
    SOperatorInfo*                    pChildOp = taosArrayGetP(pChildren, i);
5
54liuyao 已提交
1524 1525 1526
    SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
    ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
1527 1528
    closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, NULL,
                        pChInfo->aggSup.pResultBuf);
1529 1530 1531 1532 1533 1534 1535
  }
}

static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) {
  int32_t size = taosArrayGetSize(pageIds);
  for (int32_t i = 0; i < size; i++) {
    int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i);
5
54liuyao 已提交
1536
    // SFilePage* bufPage = getBufPage(pDiskBuf, pageId);
1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550
    // dBufSetBufPageRecycled(pDiskBuf, bufPage);
  }
  taosArrayClear(pageIds);
}

static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlock) {
  blockDataCleanup(pBlock);
  int32_t size = taosArrayGetSize(pWins);
  if (*index == size) {
    *index = 0;
    taosArrayClear(pWins);
    return;
  }
  blockDataEnsureCapacity(pBlock, size - *index);
1551
  uint64_t uid = 0;
1552
  for (int32_t i = *index; i < size; i++) {
H
Haojun Liao 已提交
1553
    SWinKey* pWin = taosArrayGet(pWins, i);
1554
    appendOneRow(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId);
1555
    (*index)++;
5
54liuyao 已提交
1556 1557 1558
  }
}

1559
static void destroyStateWindowOperatorInfo(void* param) {
1560
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
1561
  cleanupBasicInfo(&pInfo->binfo);
1562
  taosMemoryFreeClear(pInfo->stateKey.pData);
1563

D
dapan1121 已提交
1564
  taosMemoryFreeClear(param);
1565 1566
}

H
Haojun Liao 已提交
1567
static void freeItem(void* param) {
L
Liu Jicong 已提交
1568
  SGroupKeys* pKey = (SGroupKeys*)param;
H
Haojun Liao 已提交
1569 1570 1571
  taosMemoryFree(pKey->pData);
}

1572
void destroyIntervalOperatorInfo(void* param) {
1573
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
1574
  cleanupBasicInfo(&pInfo->binfo);
1575
  cleanupAggSup(&pInfo->aggSup);
1576 1577 1578 1579
  cleanupExprSupp(&pInfo->scalarSupp);

  tdListFree(pInfo->binfo.resultRowInfo.openWindow);

H
Haojun Liao 已提交
1580 1581 1582 1583 1584 1585 1586
  pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages);
  pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols);
  taosArrayDestroyEx(pInfo->pPrevValues, freeItem);

  pInfo->pPrevValues = NULL;
  pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins);
  pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);
1587

H
Haojun Liao 已提交
1588 1589
  cleanupGroupResInfo(&pInfo->groupResInfo);
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
D
dapan1121 已提交
1590
  taosMemoryFreeClear(param);
1591 1592
}

1593
void destroyStreamFinalIntervalOperatorInfo(void* param) {
X
Xiaoyu Wang 已提交
1594
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
1595
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
1596
  cleanupAggSup(&pInfo->aggSup);
L
Liu Jicong 已提交
1597
  // it should be empty.
5
54liuyao 已提交
1598 1599 1600
  taosHashCleanup(pInfo->pPullDataMap);
  taosArrayDestroy(pInfo->pPullWins);
  blockDataDestroy(pInfo->pPullDataRes);
1601
  taosArrayDestroy(pInfo->pRecycledPages);
H
Haojun Liao 已提交
1602
  blockDataDestroy(pInfo->pUpdateRes);
L
Liu Jicong 已提交
1603 1604
  taosArrayDestroy(pInfo->pDelWins);
  blockDataDestroy(pInfo->pDelRes);
5
54liuyao 已提交
1605

1606 1607 1608 1609
  if (pInfo->pChildren) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
1610
      destroyStreamFinalIntervalOperatorInfo(pChildOp->info);
L
Liu Jicong 已提交
1611 1612
      taosMemoryFree(pChildOp->pDownstream);
      cleanupExprSupp(&pChildOp->exprSupp);
1613 1614
      taosMemoryFreeClear(pChildOp);
    }
L
Liu Jicong 已提交
1615
    taosArrayDestroy(pInfo->pChildren);
1616
  }
1617
  nodesDestroyNode((SNode*)pInfo->pPhyNode);
5
54liuyao 已提交
1618
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
1619

D
dapan1121 已提交
1620
  taosMemoryFreeClear(param);
5
54liuyao 已提交
1621 1622
}

1623
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
5
54liuyao 已提交
1624
  for (int32_t i = 0; i < numOfCols; i++) {
5
54liuyao 已提交
1625
    if (fmIsUserDefinedFunc(pFCtx[i].functionId) || !fmIsInvertible(pFCtx[i].functionId)) {
5
54liuyao 已提交
1626 1627 1628 1629 1630 1631
      return false;
    }
  }
  return true;
}

1632
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
1633 1634
  // the primary timestamp column
  bool needed = false;
1635 1636
  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
1637

X
Xiaoyu Wang 已提交
1638
  {  // ts column
1639 1640
    SColumn c = {0};
    c.colId = 1;
1641
    c.slotId = pInfo->primaryTsIndex;
1642 1643
    c.type = TSDB_DATA_TYPE_TIMESTAMP;
    c.bytes = sizeof(int64_t);
1644
    taosArrayPush(pInfo->pInterpCols, &c);
1645 1646

    SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1647 1648 1649 1650
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = true;  // to denote no value is assigned yet
    key.pData = taosMemoryCalloc(1, c.bytes);
1651
    taosArrayPush(pInfo->pPrevValues, &key);
1652 1653
  }

X
Xiaoyu Wang 已提交
1654
  for (int32_t i = 0; i < numOfCols; ++i) {
1655 1656
    SExprInfo* pExpr = pCtx[i].pExpr;

H
Haojun Liao 已提交
1657
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1658 1659 1660
      SFunctParam* pParam = &pExpr->base.pParam[0];

      SColumn c = *pParam->pCol;
1661
      taosArrayPush(pInfo->pInterpCols, &c);
1662 1663 1664
      needed = true;

      SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1665 1666
      key.bytes = c.bytes;
      key.type = c.type;
1667
      key.isNull = false;
X
Xiaoyu Wang 已提交
1668
      key.pData = taosMemoryCalloc(1, c.bytes);
1669
      taosArrayPush(pInfo->pPrevValues, &key);
1670 1671 1672 1673 1674 1675
    }
  }

  return needed;
}

L
Liu Jicong 已提交
1676 1677
void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval,
                            int64_t waterMark) {
1678
  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
1679
    initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, waterMark);
1680 1681
    return;
  }
5
54liuyao 已提交
1682
  SStreamScanInfo* pScanInfo = downstream->info;
1683 1684 1685 1686
  pScanInfo->windowSup.parentType = type;
  pScanInfo->windowSup.pIntervalAggSup = pSup;
  pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, waterMark);
  pScanInfo->interval = *pInterval;
5
54liuyao 已提交
1687 1688
}

H
Haojun Liao 已提交
1689 1690 1691 1692 1693 1694
void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
  for (int32_t i = 0; i < numOfExpr; i++) {
    pCtx[i].isStream = true;
  }
}

1695 1696
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                          SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
L
Liu Jicong 已提交
1697 1698
                                          STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode,
                                          SExecTaskInfo* pTaskInfo, bool isStream) {
1699
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1700
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1701 1702 1703 1704
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1705
  pOperator->pTaskInfo = pTaskInfo;
L
Liu Jicong 已提交
1706
  pInfo->win = pTaskInfo->window;
1707 1708
  pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
  pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
L
Liu Jicong 已提交
1709
  pInfo->interval = *pInterval;
L
Liu Jicong 已提交
1710
  pInfo->execModel = pTaskInfo->execModel;
L
Liu Jicong 已提交
1711
  pInfo->twAggSup = *pTwAggSupp;
5
54liuyao 已提交
1712
  pInfo->ignoreExpiredData = pPhyNode->window.igExpired;
1713
  pInfo->pCondition = pPhyNode->window.node.pConditions;
1714
  pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock;
1715 1716 1717 1718

  if (pPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
L
Liu Jicong 已提交
1719
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
1720 1721 1722 1723
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
1724

1725
  pInfo->primaryTsIndex = primaryTsSlotId;
1726 1727
  SExprSupp* pSup = &pOperator->exprSupp;

1728
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
1729
  initResultSizeInfo(&pOperator->resultInfo, 4096);
1730

1731
  int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
1732 1733 1734 1735
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

1736 1737
  initBasicInfo(&pInfo->binfo, pResBlock);

1738 1739
  if (isStream) {
    ASSERT(numOfCols > 0);
H
Haojun Liao 已提交
1740
    initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
1741
  }
1742

1743
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
1744

1745
  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
X
Xiaoyu Wang 已提交
1746
  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
1747

1748
  pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, pInfo);
1749
  if (pInfo->timeWindowInterpo) {
1750
    pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
H
Haojun Liao 已提交
1751 1752 1753
    if (pInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
1754
  }
1755

1756
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
H
Haojun Liao 已提交
1757
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
1758
  pInfo->delIndex = 0;
1759
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
1760
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1761

X
Xiaoyu Wang 已提交
1762 1763 1764 1765 1766
  pOperator->name = "TimeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
1767

1768
  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL,
1769 1770
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

5
54liuyao 已提交
1771
  if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) {
L
Liu Jicong 已提交
1772 1773
    initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup, &pInfo->interval,
                           pInfo->twAggSup.waterMark);
5
54liuyao 已提交
1774 1775
  }

1776 1777 1778 1779 1780 1781 1782
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1783
_error:
1784
  destroyIntervalOperatorInfo(pInfo);
1785 1786 1787 1788 1789
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

1790
// todo handle multiple timeline cases. assume no timeline interweaving
1791 1792
static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1793
  SExprSupp*     pSup = &pOperator->exprSupp;
1794

1795
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1796 1797

  bool    masterScan = true;
1798
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813
  int64_t gid = pBlock->info.groupId;

  int64_t gap = pInfo->gap;

  if (!pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->winSup.prevTs = INT64_MIN;
  }

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

  // In case of ascending or descending order scan data, only one time window needs to be kepted for each table.
  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
1814 1815 1816
    if (gid != pRowSup->groupId || pInfo->winSup.prevTs == INT64_MIN) {
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
H
Haojun Liao 已提交
1817 1818
    } else if (((tsList[j] - pRowSup->prevTs >= 0) && (tsList[j] - pRowSup->prevTs <= gap)) ||
               ((pRowSup->prevTs - tsList[j] >= 0) && (pRowSup->prevTs - tsList[j] <= gap))) {
1819
      // The gap is less than the threshold, so it belongs to current session window that has been opened already.
1820
      doKeepTuple(pRowSup, tsList[j], gid);
1821 1822 1823 1824 1825 1826 1827 1828 1829 1830
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1831 1832
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1833
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1834
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1835 1836 1837 1838
      }

      // pInfo->numOfRows data belong to the current session window
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1839 1840
      doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                       pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1841 1842

      // here we start a new session window
1843 1844
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1845 1846 1847 1848 1849
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1850 1851
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1852
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1853
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1854 1855 1856
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
H
Haojun Liao 已提交
1857 1858
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
                   pBlock->info.rows, numOfOutput);
1859 1860
}

1861
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
1862 1863 1864 1865 1866 1867
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*          pBInfo = &pInfo->binfo;
1868
  SExprSupp*               pSup = &pOperator->exprSupp;
1869 1870

  if (pOperator->status == OP_RES_TO_RETURN) {
1871
    while (1) {
1872
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1873
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1874

1875
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1876 1877 1878 1879
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
1880

1881 1882 1883 1884 1885
      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1886
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1887 1888
  }

1889 1890 1891
  int64_t st = taosGetTimestampUs();
  int32_t order = TSDB_ORDER_ASC;

1892 1893 1894
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1895
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1896 1897 1898 1899 1900
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1901
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1902 1903
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1904 1905 1906
    doSessionWindowAggImpl(pOperator, pInfo, pBlock);
  }

1907 1908
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;

1909 1910 1911
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;

1912
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1913
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1914
  while (1) {
1915
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1916
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1917

1918
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1919 1920 1921 1922
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
1923

1924 1925 1926 1927 1928
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1929
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1930 1931
}

1932
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
H
Haojun Liao 已提交
1933
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
1934 1935
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
H
Haojun Liao 已提交
1936 1937 1938 1939 1940 1941

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);

      pkey->isNull = false;
1942
      char* val = colDataGetData(pColInfoData, rowIndex);
H
Haojun Liao 已提交
1943 1944 1945
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }
1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965

  pSliceInfo->isPrevRowSet = true;
}

static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i);

      pkey->isNull = false;
      char* val = colDataGetData(pColInfoData, rowIndex);
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }

  pSliceInfo->isNextRowSet = true;
H
Haojun Liao 已提交
1966 1967
}

1968 1969
static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex,
                             bool isLastRow) {
1970
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
dengyihao's avatar
dengyihao 已提交
1971
  bool    fillLastPoint = pSliceInfo->fillLastPoint;
1972 1973
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
1974 1975
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
    SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i);
1976 1977 1978

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994
      if (isLastRow) {
        pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
      } else if (fillLastPoint) {
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
      } else {
        pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1);

        char* val;
        val = colDataGetData(pColInfoData, rowIndex);
        memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes);
        val = colDataGetData(pColInfoData, rowIndex + 1);
        memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes);
      }
1995 1996 1997 1998

      pLinearInfo->hasNull = false;
    } else {
      pLinearInfo->hasNull = true;
1999 2000 2001
    }
  }

2002
  pSliceInfo->fillLastPoint = isLastRow ? true : false;
2003 2004
}

dengyihao's avatar
dengyihao 已提交
2005
static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) {
2006
  int32_t rows = pResBlock->info.rows;
2007
  blockDataEnsureCapacity(pResBlock, rows + 1);
2008 2009 2010
  // todo set the correct primary timestamp column

  // output the result
2011
  bool hasInterp = true;
2012 2013 2014
  for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
    SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
    int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
2015
    int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
2016

dengyihao's avatar
dengyihao 已提交
2017
    // SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
2018 2019 2020
    SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

    switch (pSliceInfo->fillType) {
2021
      case TSDB_FILL_NULL: {
2022 2023
        colDataAppendNULL(pDst, rows);
        break;
2024
      }
2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041

      case TSDB_FILL_SET_VALUE: {
        SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal;

        if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
          float v = 0;
          GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
          double v = 0;
          GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
          int64_t v = 0;
          GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        }
2042 2043
        break;
      }
2044

2045
      case TSDB_FILL_LINEAR: {
2046
        SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, srcSlot);
2047

dengyihao's avatar
dengyihao 已提交
2048 2049
        SPoint start = pLinearInfo->start;
        SPoint end = pLinearInfo->end;
2050 2051 2052
        SPoint current = {.key = pSliceInfo->current};
        current.val = taosMemoryCalloc(pLinearInfo->bytes, 1);

2053 2054
        // before interp range, do not fill
        if (start.key == INT64_MIN || end.key == INT64_MAX) {
2055
          hasInterp = false;
2056 2057 2058
          break;
        }

2059 2060 2061 2062
        if (pLinearInfo->hasNull) {
          colDataAppendNULL(pDst, rows);
        } else {
          taosGetLinearInterpolationVal(&current, pLinearInfo->type, &start, &end, pLinearInfo->type);
dengyihao's avatar
dengyihao 已提交
2063
          colDataAppend(pDst, rows, (char*)current.val, false);
2064 2065
        }

2066
        taosMemoryFree(current.val);
2067
        break;
2068
      }
2069
      case TSDB_FILL_PREV: {
2070
        if (!pSliceInfo->isPrevRowSet) {
2071
          hasInterp = false;
2072 2073 2074
          break;
        }

2075 2076
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
        colDataAppend(pDst, rows, pkey->pData, false);
2077 2078
        break;
      }
2079 2080

      case TSDB_FILL_NEXT: {
2081
        if (!pSliceInfo->isNextRowSet) {
2082
          hasInterp = false;
2083 2084 2085
          break;
        }

2086 2087
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot);
        colDataAppend(pDst, rows, pkey->pData, false);
2088 2089
        break;
      }
2090 2091 2092 2093 2094 2095

      case TSDB_FILL_NONE:
      default:
        break;
    }
  }
2096 2097 2098 2099

  if (hasInterp) {
    pResBlock->info.rows += 1;
  }
2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111
}

static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pPrevRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pPrevRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

2112
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pPrevRow, &key);
  }

2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152
  pInfo->isPrevRowSet = false;

  return TSDB_CODE_SUCCESS;
}

static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pNextRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pNextRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pNextRow, &key);
  }

  pInfo->isNextRowSet = false;

2153 2154 2155
  return TSDB_CODE_SUCCESS;
}

2156 2157 2158 2159 2160 2161
static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pLinearInfo != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo));
2162
  if (pInfo->pLinearInfo == NULL) {
2163 2164 2165 2166 2167 2168 2169
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

2170 2171
    SFillLinearInfo linearInfo = {0};
    linearInfo.start.key = INT64_MIN;
dengyihao's avatar
dengyihao 已提交
2172
    linearInfo.end.key = INT64_MAX;
2173
    linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes);
dengyihao's avatar
dengyihao 已提交
2174 2175 2176
    linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes);
    linearInfo.hasNull = false;
    linearInfo.type = pColInfo->info.type;
2177 2178
    linearInfo.bytes = pColInfo->info.bytes;
    taosArrayPush(pInfo->pLinearInfo, &linearInfo);
2179 2180
  }

2181 2182
  pInfo->fillLastPoint = false;

2183 2184 2185
  return TSDB_CODE_SUCCESS;
}

2186 2187 2188 2189
static bool needToFillLastPoint(STimeSliceOperatorInfo* pSliceInfo) {
  return (pSliceInfo->fillLastPoint == true && pSliceInfo->fillType == TSDB_FILL_LINEAR);
}

2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206
static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  int32_t code;
  code = initPrevRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initNextRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initFillLinearInfo(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

2207 2208 2209
  return TSDB_CODE_SUCCESS;
}

H
Haojun Liao 已提交
2210
static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
2211 2212 2213 2214
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

2215 2216
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

2217
  STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
2218 2219
  SSDataBlock*            pResBlock = pSliceInfo->pRes;
  SExprSupp*              pSup = &pOperator->exprSupp;
H
Haojun Liao 已提交
2220

2221 2222
  //  if (pOperator->status == OP_RES_TO_RETURN) {
  //    //    doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes);
2223
  //    if (pResBlock->info.rows == 0 || !hasRemainResults(&pSliceInfo->groupResInfo)) {
2224 2225 2226 2227 2228
  //      doSetOperatorCompleted(pOperator);
  //    }
  //
  //    return pResBlock;
  //  }
2229

2230 2231
  int32_t        order = TSDB_ORDER_ASC;
  SInterval*     pInterval = &pSliceInfo->interval;
2232 2233
  SOperatorInfo* downstream = pOperator->pDownstream[0];

2234 2235
  blockDataCleanup(pResBlock);

2236
  while (1) {
2237
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
2238 2239 2240 2241
    if (pBlock == NULL) {
      break;
    }

2242
    int32_t code = initKeeperInfo(pSliceInfo, pBlock);
2243
    if (code != TSDB_CODE_SUCCESS) {
2244
      T_LONG_JMP(pTaskInfo->env, code);
2245 2246
    }

2247
    // the pDataBlock are always the same one, no need to call this again
2248
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
H
Haojun Liao 已提交
2249

2250
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
2251 2252
    for (int32_t i = 0; i < pBlock->info.rows; ++i) {
      int64_t ts = *(int64_t*)colDataGetData(pTsCol, i);
H
Haojun Liao 已提交
2253

dengyihao's avatar
dengyihao 已提交
2254
      if (i == 0 && needToFillLastPoint(pSliceInfo)) {  // first row in current block
2255 2256 2257 2258 2259 2260 2261 2262 2263
        doKeepLinearInfo(pSliceInfo, pBlock, i, false);
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
          genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
        }
2264
      }
2265

2266 2267 2268
      if (pSliceInfo->current > pSliceInfo->win.ekey) {
        doSetOperatorCompleted(pOperator);
        break;
2269 2270
      }

H
Haojun Liao 已提交
2271
      if (ts == pSliceInfo->current) {
2272
        for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
2273
          SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
2274 2275
          int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
          int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
H
Haojun Liao 已提交
2276 2277

          SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
2278
          SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
H
Haojun Liao 已提交
2279

2280 2281 2282 2283 2284
          if (colDataIsNull_s(pSrc, i)) {
            colDataAppendNULL(pDst, pResBlock->info.rows);
            continue;
          }

H
Haojun Liao 已提交
2285
          char* v = colDataGetData(pSrc, i);
2286
          colDataAppend(pDst, pResBlock->info.rows, v, false);
H
Haojun Liao 已提交
2287 2288
        }

2289
        pResBlock->info.rows += 1;
2290
        doKeepPrevRows(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2291

2292 2293
        // for linear interpolation, always fill value between this and next points;
        // if its the first point in data block, also fill values between previous(if there's any) and this point;
2294 2295
        // if its the last point in data block, no need to fill, but reserve this point as the start value and do
        // the interpolation when processing next data block.
2296
        if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2297 2298
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2299
          if (i < pBlock->info.rows - 1) {
2300
            doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2301 2302 2303
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2304
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2305 2306
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2307 2308 2309 2310
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }
2311

2312 2313 2314 2315 2316
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
            }
dengyihao's avatar
dengyihao 已提交
2317 2318
          } else {  // it is the last row of current block
            // store ts value as start, and calculate interp value when processing next block
2319
            doKeepLinearInfo(pSliceInfo, pBlock, i, true);
2320
          }
dengyihao's avatar
dengyihao 已提交
2321
        } else {  // non-linear interpolation
2322 2323 2324 2325 2326 2327
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (pSliceInfo->current > pSliceInfo->win.ekey) {
            doSetOperatorCompleted(pOperator);
            break;
          }
2328

2329 2330 2331
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
2332
        }
H
Haojun Liao 已提交
2333
      } else if (ts < pSliceInfo->current) {
2334
        // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate
2335 2336
        doKeepPrevRows(pSliceInfo, pBlock, i);

2337
        if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2338
          // no need to increate pSliceInfo->current here
dengyihao's avatar
dengyihao 已提交
2339
          // pSliceInfo->current =
2340 2341
          //    taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (i < pBlock->info.rows - 1) {
2342
            doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2343 2344 2345
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2346
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2347 2348
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2349 2350 2351 2352 2353 2354 2355
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }

              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
H
Haojun Liao 已提交
2356 2357
                break;
              }
H
Haojun Liao 已提交
2358
            }
2359
          }
dengyihao's avatar
dengyihao 已提交
2360
        } else {  // non-linear interpolation
2361 2362 2363 2364 2365 2366
          if (i < pBlock->info.rows - 1) {
            // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
            doKeepNextRows(pSliceInfo, pBlock, i + 1);
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2367
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2368 2369
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2370 2371 2372 2373
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }
2374

2375 2376 2377 2378 2379 2380
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
            } else {
              // ignore current row, and do nothing
H
Haojun Liao 已提交
2381
            }
2382 2383
          } else {  // it is the last row of current block
            doKeepPrevRows(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2384
          }
2385 2386
        }
      } else {  // ts > pSliceInfo->current
2387
        // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
2388 2389
        doKeepNextRows(pSliceInfo, pBlock, i);

2390
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
2391
          genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
2392 2393
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2394 2395 2396
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
2397 2398
        }

2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415
        // add current row if timestamp match
        if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
          for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
            SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
            int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
            int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;

            SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
            SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

            char* v = colDataGetData(pSrc, i);
            colDataAppend(pDst, pResBlock->info.rows, v, false);
          }

          pResBlock->info.rows += 1;
          doKeepPrevRows(pSliceInfo, pBlock, i);

2416 2417 2418 2419
          if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
            pSliceInfo->current =
                taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
            if (i < pBlock->info.rows - 1) {
2420
              doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2421 2422 2423
              int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
              if (nextTs > pSliceInfo->current) {
                while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2424
                  genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2425 2426
                  pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                    pInterval->precision);
2427 2428 2429 2430
                  if (pResBlock->info.rows >= pResBlock->info.capacity) {
                    break;
                  }
                }
2431

2432 2433 2434 2435 2436 2437
                if (pSliceInfo->current > pSliceInfo->win.ekey) {
                  doSetOperatorCompleted(pOperator);
                  break;
                }
              }
            }
dengyihao's avatar
dengyihao 已提交
2438
          } else {  // non-linear interpolation
2439 2440 2441 2442 2443 2444
            pSliceInfo->current =
                taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);

            if (pResBlock->info.rows >= pResBlock->info.capacity) {
              break;
            }
2445 2446 2447
          }
        }

2448 2449 2450
        if (pSliceInfo->current > pSliceInfo->win.ekey) {
          doSetOperatorCompleted(pOperator);
          break;
H
Haojun Liao 已提交
2451 2452 2453
        }
      }
    }
2454
  }
2455

2456 2457
  // check if need to interpolate after last datablock
  // except for fill(next), fill(linear)
dengyihao's avatar
dengyihao 已提交
2458 2459
  while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT &&
         pSliceInfo->fillType != TSDB_FILL_LINEAR) {
2460 2461 2462 2463 2464
    genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
    pSliceInfo->current =
        taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
    if (pResBlock->info.rows >= pResBlock->info.capacity) {
      break;
2465
    }
2466 2467 2468 2469
  }

  // restore the value
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
2470
  if (pResBlock->info.rows == 0) {
2471 2472 2473
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
2474 2475 2476
  return pResBlock->info.rows == 0 ? NULL : pResBlock;
}

2477
void destroyTimeSliceOperatorInfo(void* param) {
2478 2479 2480 2481 2482 2483 2484 2485
  STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param;

  pInfo->pRes = blockDataDestroy(pInfo->pRes);

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) {
    SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i);
    taosMemoryFree(pKey->pData);
  }
2486
  taosArrayDestroy(pInfo->pPrevRow);
2487 2488 2489 2490 2491 2492

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pNextRow); ++i) {
    SGroupKeys* pKey = taosArrayGet(pInfo->pNextRow, i);
    taosMemoryFree(pKey->pData);
  }
  taosArrayDestroy(pInfo->pNextRow);
2493 2494 2495 2496 2497 2498

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pLinearInfo); ++i) {
    SFillLinearInfo* pKey = taosArrayGet(pInfo->pLinearInfo, i);
    taosMemoryFree(pKey->start.val);
    taosMemoryFree(pKey->end.val);
  }
2499 2500
  taosArrayDestroy(pInfo->pLinearInfo);

2501
  taosMemoryFree(pInfo->pFillColInfo);
2502 2503 2504
  taosMemoryFreeClear(param);
}

2505
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
2506 2507 2508 2509 2510 2511
  STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pOperator == NULL || pInfo == NULL) {
    goto _error;
  }

2512
  SInterpFuncPhysiNode* pInterpPhyNode = (SInterpFuncPhysiNode*)pPhyNode;
2513
  SExprSupp*            pSup = &pOperator->exprSupp;
2514

2515
  int32_t    numOfExprs = 0;
2516
  SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
2517
  int32_t    code = initExprSupp(pSup, pExprInfo, numOfExprs);
H
Haojun Liao 已提交
2518 2519 2520 2521
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2522
  if (pInterpPhyNode->pExprs != NULL) {
2523
    int32_t    num = 0;
2524 2525 2526 2527 2528 2529 2530 2531 2532
    SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

  pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries);
  pInfo->fillType = convertFillType(pInterpPhyNode->fillMode);
2533
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2534

H
Haojun Liao 已提交
2535
  pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, NULL, 0, (SNodeListNode*)pInterpPhyNode->pFillValues);
2536
  pInfo->pLinearInfo = NULL;
L
Liu Jicong 已提交
2537 2538
  pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  pInfo->win = pInterpPhyNode->timeRange;
2539
  pInfo->interval.interval = pInterpPhyNode->interval;
L
Liu Jicong 已提交
2540
  pInfo->current = pInfo->win.skey;
H
Haojun Liao 已提交
2541

2542
  pOperator->name = "TimeSliceOperator";
2543
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC;
2544 2545 2546 2547
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
  pOperator->pTaskInfo = pTaskInfo;
2548

2549
  pOperator->fpSet =
2550
      createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyTimeSliceOperatorInfo, NULL, NULL, NULL);
2551

2552 2553
  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);

H
Haojun Liao 已提交
2554
  code = appendDownstream(pOperator, &downstream, 1);
2555 2556
  return pOperator;

L
Liu Jicong 已提交
2557
_error:
2558 2559 2560 2561 2562 2563
  taosMemoryFree(pInfo);
  taosMemoryFree(pOperator);
  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
  return NULL;
}

2564 2565
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
                                             SExecTaskInfo* pTaskInfo) {
2566 2567 2568 2569 2570 2571
  SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

L
Liu Jicong 已提交
2572
  int32_t      num = 0;
2573 2574 2575 2576 2577 2578 2579
  SExprInfo*   pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc);
  int32_t      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;

  SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
2580 2581 2582
  pInfo->stateKey.type = pInfo->stateCol.type;
  pInfo->stateKey.bytes = pInfo->stateCol.bytes;
  pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
2583
  pInfo->pCondition = pStateNode->window.node.pConditions;
2584 2585 2586 2587
  if (pInfo->stateKey.pData == NULL) {
    goto _error;
  }

2588 2589
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

2590
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2591
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
2592 2593 2594 2595
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2596
  initBasicInfo(&pInfo->binfo, pResBlock);
2597
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2598

L
Liu Jicong 已提交
2599 2600 2601
  pInfo->twAggSup =
      (STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType};
  ;
2602 2603
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

X
Xiaoyu Wang 已提交
2604 2605
  pInfo->tsSlotId = tsSlotId;
  pOperator->name = "StateWindowOperator";
2606
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE;
X
Xiaoyu Wang 已提交
2607 2608 2609 2610
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
2611 2612 2613 2614

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL,
                                         destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

2615 2616 2617 2618 2619
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2620 2621
  return pOperator;

L
Liu Jicong 已提交
2622
_error:
2623 2624 2625
  destroyStateWindowOperatorInfo(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
2626 2627 2628
  return NULL;
}

2629
void destroySWindowOperatorInfo(void* param) {
2630
  SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
2631 2632 2633
  if (pInfo == NULL) {
    return;
  }
2634

2635
  cleanupBasicInfo(&pInfo->binfo);
H
Haojun Liao 已提交
2636 2637 2638 2639
  colDataDestroy(&pInfo->twAggSup.timeWindowData);

  cleanupAggSup(&pInfo->aggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
D
dapan1121 已提交
2640
  taosMemoryFreeClear(param);
2641 2642
}

H
Haojun Liao 已提交
2643
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode,
2644
                                            SExecTaskInfo* pTaskInfo) {
2645 2646 2647 2648 2649 2650
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

2651
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
2652
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2653

2654
  int32_t      numOfCols = 0;
H
Haojun Liao 已提交
2655 2656 2657
  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc);

2658
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
2659 2660 2661 2662
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2663 2664
  initBasicInfo(&pInfo->binfo, pResBlock);

H
Haojun Liao 已提交
2665 2666 2667 2668
  pInfo->twAggSup.waterMark = pSessionNode->window.watermark;
  pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType;
  pInfo->gap = pSessionNode->gap;

2669
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2670 2671
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

2672
  pInfo->tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
L
Liu Jicong 已提交
2673 2674 2675
  pInfo->binfo.pRes = pResBlock;
  pInfo->winSup.prevTs = INT64_MIN;
  pInfo->reptScan = false;
2676
  pInfo->pCondition = pSessionNode->window.node.pConditions;
H
Haojun Liao 已提交
2677

L
Liu Jicong 已提交
2678
  pOperator->name = "SessionWindowAggOperator";
2679
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
2680
  pOperator->blocking = true;
L
Liu Jicong 已提交
2681 2682
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
2683 2684 2685 2686 2687

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL,
                                         destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  pOperator->pTaskInfo = pTaskInfo;
  code = appendDownstream(pOperator, &downstream, 1);
2688 2689 2690 2691
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2692 2693
  return pOperator;

L
Liu Jicong 已提交
2694
_error:
2695
  destroySWindowOperatorInfo(pInfo);
2696 2697 2698
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
2699
}
5
54liuyao 已提交
2700

5
54liuyao 已提交
2701
void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
2702
                      SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) {
5
54liuyao 已提交
2703 2704
  for (int32_t k = 0; k < numOfOutput; ++k) {
    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
2705 2706 2707 2708 2709
      if (!pTimeWindowData) {
        continue;
      }

      SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);
L
Liu Jicong 已提交
2710 2711
      char*                p = GET_ROWCELL_INTERBUF(pEntryInfo);
      SColumnInfoData      idata = {0};
2712 2713 2714 2715 2716 2717 2718 2719
      idata.info.type = TSDB_DATA_TYPE_BIGINT;
      idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
      idata.pData = p;

      SScalarParam out = {.columnData = &idata};
      SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
      pDestCtx[k].sfp.process(&tw, 1, &out);
      pEntryInfo->numOfRes = 1;
L
Liu Jicong 已提交
2720
    } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
2721
      int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
5
54liuyao 已提交
2722 2723 2724
      if (code != TSDB_CODE_SUCCESS) {
        qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
        pTaskInfo->code = code;
2725
        T_LONG_JMP(pTaskInfo->env, code);
5
54liuyao 已提交
2726 2727 2728 2729 2730
      }
    }
  }
}

2731 2732 2733 2734
bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) {
  int32_t bytes = sizeof(TSKEY);
  SET_RES_WINDOW_KEY(pSup->keyBuf, &ts, bytes, groupId);
  SResultRowPosition* p1 =
2735
      (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
2736 2737 2738
  return p1 != NULL;
}

2739 2740 2741 2742 2743 2744
STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
  return w;
}

2745
static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray,
L
Liu Jicong 已提交
2746 2747
                                  int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo,
                                  SHashObj* pUpdatedMap) {
5
54liuyao 已提交
2748
  int32_t size = taosArrayGetSize(pWinArray);
5
54liuyao 已提交
2749 2750 2751
  if (!pInfo->pChildren) {
    return;
  }
5
54liuyao 已提交
2752
  for (int32_t i = 0; i < size; i++) {
H
Haojun Liao 已提交
2753
    SWinKey*    pWinRes = taosArrayGet(pWinArray, i);
2754
    SResultRow* pCurResult = NULL;
2755 2756 2757 2758 2759
    STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval);
    if (isDeletedWindow(&parentWin, pWinRes->groupId, &pInfo->aggSup) && isCloseWindow(&parentWin, &pInfo->twAggSup)) {
      continue;
    }
    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &parentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx,
2760
                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2761
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
2762
    int32_t num = 0;
5
54liuyao 已提交
2763 2764 2765
    for (int32_t j = 0; j < numOfChildren; j++) {
      SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
2766
      SExprSupp*                pChildSup = &pChildOp->exprSupp;
2767 2768 2769
      if (!hasIntervalWindow(&pChInfo->aggSup, pWinRes->ts, pWinRes->groupId)) {
        continue;
      }
2770
      num++;
2771
      SResultRow* pChResult = NULL;
2772
      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &parentWin, true, &pChResult, pWinRes->groupId,
2773 2774
                             pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup,
                             pTaskInfo);
2775 2776
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true);
      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
2777
    }
2778 2779
    if (num > 1 && pUpdatedMap) {
      saveWinResultRow(pCurResult, pWinRes->groupId, pUpdatedMap);
D
dapan1121 已提交
2780
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur);
2781
    }
5
54liuyao 已提交
2782 2783 2784 2785 2786
  }
}

bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
  SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId);
2787
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
L
Liu Jicong 已提交
2788
                                                               GET_RES_WINDOW_KEY_LEN(sizeof(int64_t)));
5
54liuyao 已提交
2789 2790 2791
  return p1 == NULL;
}

L
Liu Jicong 已提交
2792 2793 2794 2795
int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,
                        STimeWindow* pNextWin) {
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2796 2797 2798 2799
  int32_t prevEndPos = forwardRows - 1 + startPos;
  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
}

H
Haojun Liao 已提交
2800
void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {
5
54liuyao 已提交
2801 2802 2803 2804
  SArray* childIds = taosArrayInit(8, sizeof(int32_t));
  for (int32_t i = 0; i < size; i++) {
    taosArrayPush(childIds, &i);
  }
H
Haojun Liao 已提交
2805
  taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*));
5
54liuyao 已提交
2806 2807 2808 2809
}

static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

5
54liuyao 已提交
2810
static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
5
54liuyao 已提交
2811
                           SHashObj* pUpdatedMap) {
5
54liuyao 已提交
2812
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
X
Xiaoyu Wang 已提交
2813 2814
  SResultRowInfo*                   pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*                    pTaskInfo = pOperatorInfo->pTaskInfo;
2815 2816
  SExprSupp*                        pSup = &pOperatorInfo->exprSupp;
  int32_t                           numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
2817 2818 2819 2820 2821
  int32_t                           step = 1;
  bool                              ascScan = true;
  TSKEY*                            tsCols = NULL;
  SResultRow*                       pResult = NULL;
  int32_t                           forwardRows = 0;
5
54liuyao 已提交
2822

5
54liuyao 已提交
2823 2824 2825
  ASSERT(pSDataBlock->pDataBlock != NULL);
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
2826

X
Xiaoyu Wang 已提交
2827 2828
  int32_t     startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
5
54liuyao 已提交
2829 2830 2831 2832 2833 2834
  STimeWindow nextWin = {0};
  if (IS_FINAL_OP(pInfo)) {
    nextWin = getFinalTimeWindow(ts, &pInfo->interval);
  } else {
    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order);
  }
5
54liuyao 已提交
2835
  while (1) {
5
54liuyao 已提交
2836
    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
5
54liuyao 已提交
2837
    if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
5
54liuyao 已提交
2838 2839 2840 2841 2842 2843 2844
      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
      if (startPos < 0) {
        break;
      }
      continue;
    }
    if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) {
L
Liu Jicong 已提交
2845
      bool    ignore = true;
H
Haojun Liao 已提交
2846
      SWinKey winRes = {
L
Liu Jicong 已提交
2847 2848 2849
          .ts = nextWin.skey,
          .groupId = tableGroupId,
      };
H
Haojun Liao 已提交
2850
      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
2851 2852 2853
      if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId};
        // add pull data request
5
54liuyao 已提交
2854
        savePullWindow(&pull, pInfo->pPullWins);
5
54liuyao 已提交
2855 2856 2857
        int32_t size = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, &winRes, size);
        qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size);
5
54liuyao 已提交
2858 2859 2860
      } else {
        int32_t index = -1;
        SArray* chArray = NULL;
5
54liuyao 已提交
2861
        int32_t chId = 0;
5
54liuyao 已提交
2862
        if (chIds) {
L
Liu Jicong 已提交
2863
          chArray = *(void**)chIds;
5
54liuyao 已提交
2864
          chId = getChildIndex(pSDataBlock);
5
54liuyao 已提交
2865 2866
          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
        }
L
Liu Jicong 已提交
2867
        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {
5
54liuyao 已提交
2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878
          ignore = false;
        }
      }

      if (ignore) {
        startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
        if (startPos < 0) {
          break;
        }
        continue;
      }
5
54liuyao 已提交
2879
    }
5
54liuyao 已提交
2880

2881 2882
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx,
                                          numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2883
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
2884
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
2885
    }
5
54liuyao 已提交
2886

5
54liuyao 已提交
2887 2888 2889
    if (IS_FINAL_OP(pInfo)) {
      forwardRows = 1;
    } else {
H
Hongze Cheng 已提交
2890 2891
      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,
                                             NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2892
    }
5
54liuyao 已提交
2893 2894
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
D
dapan1121 已提交
2895
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
5
54liuyao 已提交
2896
    }
5
54liuyao 已提交
2897
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
2898 2899
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pSDataBlock->info.rows, numOfOutput);
2900
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
2901
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
5
54liuyao 已提交
2902 2903 2904 2905 2906 2907 2908
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
    if (startPos < 0) {
      break;
    }
  }
}

5
54liuyao 已提交
2909
static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) {
2910
  tSimpleHashClear(pInfo->aggSup.pResultRowHashTable);
5
54liuyao 已提交
2911
  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
2912
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
L
Liu Jicong 已提交
2913
  pInfo->aggSup.currentPageId = -1;
5
54liuyao 已提交
2914 2915
}

5
54liuyao 已提交
2916 2917 2918 2919
static void clearSpecialDataBlock(SSDataBlock* pBlock) {
  if (pBlock->info.rows <= 0) {
    return;
  }
5
54liuyao 已提交
2920 2921 2922
  blockDataCleanup(pBlock);
}

2923
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
5
54liuyao 已提交
2924 2925
  // ASSERT(pDest->info.capacity >= pSource->info.rows);
  blockDataEnsureCapacity(pDest, pSource->info.rows);
5
54liuyao 已提交
2926
  clearSpecialDataBlock(pDest);
5
54liuyao 已提交
2927 2928
  SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
  SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
2929

5
54liuyao 已提交
2930
  // copy timestamp column
2931 2932
  colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info);
  for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) {
5
54liuyao 已提交
2933 2934 2935
    SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
    colDataAppendNNULL(pCol, 0, pSource->info.rows);
  }
2936

5
54liuyao 已提交
2937
  pDest->info.rows = pSource->info.rows;
2938 2939
  pDest->info.groupId = pSource->info.groupId;
  pDest->info.type = pSource->info.type;
5
54liuyao 已提交
2940 2941 2942
  blockDataUpdateTsWindow(pDest, 0);
}

5
54liuyao 已提交
2943 2944 2945 2946 2947 2948
static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
  clearSpecialDataBlock(pBlock);
  int32_t size = taosArrayGetSize(array);
  if (size - (*pIndex) == 0) {
    return;
  }
L
Liu Jicong 已提交
2949
  blockDataEnsureCapacity(pBlock, size - (*pIndex));
5
54liuyao 已提交
2950
  ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
2951 2952 2953 2954 2955
  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
5
54liuyao 已提交
2956
  for (; (*pIndex) < size; (*pIndex)++) {
L
Liu Jicong 已提交
2957
    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));
5
54liuyao 已提交
2958 2959 2960
    colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
    colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
2961 2962
    colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
5
54liuyao 已提交
2963 2964 2965 2966 2967 2968 2969 2970 2971
    pBlock->info.rows++;
  }
  if ((*pIndex) == size) {
    *pIndex = 0;
    taosArrayClear(array);
  }
  blockDataUpdateTsWindow(pBlock, 0);
}

L
Liu Jicong 已提交
2972
void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
5
54liuyao 已提交
2973
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
L
Liu Jicong 已提交
2974
  TSKEY*           tsData = (TSKEY*)pStartCol->pData;
5
54liuyao 已提交
2975
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
L
Liu Jicong 已提交
2976 2977
  uint64_t*        groupIdData = (uint64_t*)pGroupCol->pData;
  int32_t          chId = getChildIndex(pBlock);
5
54liuyao 已提交
2978
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
H
Haojun Liao 已提交
2979 2980
    SWinKey winRes = {.ts = tsData[i], .groupId = groupIdData[i]};
    void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
2981
    if (chIds) {
L
Liu Jicong 已提交
2982
      SArray* chArray = *(SArray**)chIds;
5
54liuyao 已提交
2983 2984
      int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
      if (index != -1) {
5
54liuyao 已提交
2985
        qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId);
5
54liuyao 已提交
2986 2987 2988
        taosArrayRemove(chArray, index);
        if (taosArrayGetSize(chArray) == 0) {
          // pull data is over
H
Haojun Liao 已提交
2989
          taosHashRemove(pMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
2990 2991 2992 2993 2994
        }
      }
    }
  }
}
5
54liuyao 已提交
2995

2996 2997 2998
static void addRetriveWindow(SArray* wins, SStreamFinalIntervalOperatorInfo* pInfo) {
  int32_t size = taosArrayGetSize(wins);
  for (int32_t i = 0; i < size; i++) {
L
Liu Jicong 已提交
2999
    SWinKey*    winKey = taosArrayGet(wins, i);
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014
    STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
    if (isCloseWindow(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) {
      void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
      if (!chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId};
        // add pull data request
        savePullWindow(&pull, pInfo->pPullWins);
        int32_t size = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, winKey, size);
        qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, size);
      }
    }
  }
}

5
54liuyao 已提交
3015 3016 3017 3018 3019 3020
static void clearFunctionContext(SExprSupp* pSup) {
  for (int32_t i = 0; i < pSup->numOfExprs; i++) {
    pSup->pCtx[i].saveHandle.currentPage = -1;
  }
}

5
54liuyao 已提交
3021 3022
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
  SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
3023 3024 3025 3026 3027 3028

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  SArray*        pUpdated = taosArrayInit(4, POINTER_BYTES);
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
  TSKEY          maxTs = INT64_MIN;
5
54liuyao 已提交
3029

3030 3031
  SExprSupp* pSup = &pOperator->exprSupp;

5
54liuyao 已提交
3032
  qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
L
Liu Jicong 已提交
3033

5
54liuyao 已提交
3034 3035 3036
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
3037 3038 3039 3040
    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
    if (pInfo->pPullDataRes->info.rows != 0) {
      // process the rest of the data
      ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3041
      printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3042 3043 3044
      return pInfo->pPullDataRes;
    }

3045 3046 3047 3048 3049 3050 3051
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
      return pInfo->pDelRes;
    }

5
54liuyao 已提交
3052
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3053
    if (pInfo->binfo.pRes->info.rows == 0) {
5
54liuyao 已提交
3054
      pOperator->status = OP_EXEC_DONE;
5
54liuyao 已提交
3055
      if (!IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
3056
        clearFunctionContext(&pOperator->exprSupp);
5
54liuyao 已提交
3057 3058
        // semi interval operator clear disk buffer
        clearStreamIntervalOperator(pInfo);
3059
        qDebug("===stream===clear semi operator");
3060 3061
      } else {
        freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3062
      }
5
54liuyao 已提交
3063 3064
      return NULL;
    }
5
54liuyao 已提交
3065
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3066 3067
    return pInfo->binfo.pRes;
  } else {
5
54liuyao 已提交
3068 3069 3070
    if (!IS_FINAL_OP(pInfo)) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
      if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3071
        printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3072 3073
        return pInfo->binfo.pRes;
      }
5
54liuyao 已提交
3074 3075 3076 3077
    }
    if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
      pInfo->returnUpdate = false;
      ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3078
      printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3079 3080
      // process the rest of the data
      return pInfo->pUpdateRes;
5
54liuyao 已提交
3081
    }
3082 3083 3084
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
5
54liuyao 已提交
3085
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
3086 3087
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
3088 3089 3090 3091 3092
  }

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
3093
      clearSpecialDataBlock(pInfo->pUpdateRes);
5
54liuyao 已提交
3094
      removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
5
54liuyao 已提交
3095
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
3096
      qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3097 3098
      break;
    }
5
54liuyao 已提交
3099
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv");
5
54liuyao 已提交
3100
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
3101
    maxTs = TMAX(maxTs, pBlock->info.watermark);
3102

H
Haojun Liao 已提交
3103 3104
    ASSERT(pBlock->info.type != STREAM_INVERT);
    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
5
54liuyao 已提交
3105 3106
      pInfo->binfo.pRes->info.type = pBlock->info.type;
    } else if (pBlock->info.type == STREAM_CLEAR) {
H
Haojun Liao 已提交
3107
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinKey));
3108
      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
5
54liuyao 已提交
3109
      if (IS_FINAL_OP(pInfo)) {
L
Liu Jicong 已提交
3110 3111
        int32_t                           childIndex = getChildIndex(pBlock);
        SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
3112
        SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
L
Liu Jicong 已提交
3113
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;
3114

3115 3116 3117
        doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildSup->numOfExprs, pBlock, NULL);
        rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs,
                              pOperator->pTaskInfo, NULL);
5
54liuyao 已提交
3118 3119
        taosArrayDestroy(pUpWins);
        continue;
3120
      }
5
54liuyao 已提交
3121
      removeResults(pUpWins, pUpdatedMap);
3122
      copyDataBlock(pInfo->pUpdateRes, pBlock);
5
54liuyao 已提交
3123
      pInfo->returnUpdate = true;
3124
      taosArrayDestroy(pUpWins);
5
54liuyao 已提交
3125
      break;
3126
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
3127 3128
      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, delWins, &pInfo->interval, pUpdatedMap);
3129 3130 3131 3132 3133
      if (IS_FINAL_OP(pInfo)) {
        int32_t                           childIndex = getChildIndex(pBlock);
        SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;
3134
        doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval, NULL);
L
Liu Jicong 已提交
3135 3136
        rebuildIntervalWindow(pInfo, pSup, delWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs,
                              pOperator->pTaskInfo, pUpdatedMap);
3137 3138 3139
        addRetriveWindow(delWins, pInfo);
        taosArrayAddAll(pInfo->pDelWins, delWins);
        taosArrayDestroy(delWins);
3140 3141
        continue;
      }
3142 3143 3144
      removeResults(delWins, pUpdatedMap);
      taosArrayAddAll(pInfo->pDelWins, delWins);
      taosArrayDestroy(delWins);
3145
      break;
5
54liuyao 已提交
3146
    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
3147
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
5
54liuyao 已提交
3148
      continue;
5
54liuyao 已提交
3149
    } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
H
Haojun Liao 已提交
3150
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinKey));
5
54liuyao 已提交
3151
      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
5
54liuyao 已提交
3152
      removeResults(pUpWins, pUpdatedMap);
5
54liuyao 已提交
3153 3154 3155 3156 3157
      taosArrayDestroy(pUpWins);
      if (taosArrayGetSize(pUpdated) > 0) {
        break;
      }
      continue;
L
Liu Jicong 已提交
3158 3159
    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) {
      processPullOver(pBlock, pInfo->pPullDataMap);
5
54liuyao 已提交
3160
      continue;
5
54liuyao 已提交
3161
    }
5
54liuyao 已提交
3162

5
54liuyao 已提交
3163 3164 3165 3166
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
3167
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
5
54liuyao 已提交
3168
    doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
5
54liuyao 已提交
3169
    if (IS_FINAL_OP(pInfo)) {
S
shenglian zhou 已提交
3170
      int32_t chIndex = getChildIndex(pBlock);
5
54liuyao 已提交
3171 3172 3173 3174 3175
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
        SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
        if (!pChildOp) {
3176
          T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3177
        }
3178 3179
        SStreamFinalIntervalOperatorInfo* pTmpInfo = pChildOp->info;
        pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3180
        taosArrayPush(pInfo->pChildren, &pChildOp);
5
54liuyao 已提交
3181
        qDebug("===stream===add child, id:%d", chIndex);
5
54liuyao 已提交
3182
      }
S
shenglian zhou 已提交
3183
      SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
3184
      SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
3185
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
5
54liuyao 已提交
3186 3187 3188
      doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL);
    }
  }
S
shenglian zhou 已提交
3189

5
54liuyao 已提交
3190
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
3191
  if (IS_FINAL_OP(pInfo)) {
3192
    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap,
5
54liuyao 已提交
3193
                        pUpdatedMap, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3194
    closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
3195 3196
  } else {
    pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;
5
54liuyao 已提交
3197 3198
  }

5
54liuyao 已提交
3199 3200 3201 3202 3203 3204 3205
  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosHashCleanup(pUpdatedMap);
  taosArraySort(pUpdated, resultrowComparAsc);

3206
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
3207 3208
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
3209 3210 3211 3212 3213

  doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
  if (pInfo->pPullDataRes->info.rows != 0) {
    // process the rest of the data
    ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3214
    printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3215 3216 3217
    return pInfo->pPullDataRes;
  }

3218
  // we should send result first.
5
54liuyao 已提交
3219
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3220
  if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3221
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3222 3223 3224 3225 3226 3227
    return pInfo->binfo.pRes;
  }

  if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
    pInfo->returnUpdate = false;
    ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3228
    printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3229 3230 3231
    // process the rest of the data
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
3232

3233 3234 3235
  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows != 0) {
    // process the rest of the data
5
54liuyao 已提交
3236
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
3237 3238
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
3239 3240 3241
  return NULL;
}

S
shenglian zhou 已提交
3242 3243 3244
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                     SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
  SIntervalPhysiNode*               pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
5
54liuyao 已提交
3245
  SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo));
S
shenglian zhou 已提交
3246
  SOperatorInfo*                    pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3247 3248 3249
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
3250

3251
  pOperator->pTaskInfo = pTaskInfo;
5
54liuyao 已提交
3252
  pInfo->order = TSDB_ORDER_ASC;
S
shenglian zhou 已提交
3253 3254 3255 3256 3257 3258 3259 3260
  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                .sliding = pIntervalPhyNode->sliding,
                                .intervalUnit = pIntervalPhyNode->intervalUnit,
                                .slidingUnit = pIntervalPhyNode->slidingUnit,
                                .offset = pIntervalPhyNode->offset,
                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pIntervalPhyNode->window.watermark,
5
54liuyao 已提交
3261 3262
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
S
shenglian zhou 已提交
3263
  };
3264
  ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY);
5
54liuyao 已提交
3265 3266
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
3267
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3268 3269 3270 3271 3272 3273 3274 3275 3276
  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

S
shenglian zhou 已提交
3277 3278
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5
54liuyao 已提交
3279
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
3280 3281

  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
3282 3283 3284 3285
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
3286
  initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs);
3287 3288
  initBasicInfo(&pInfo->binfo, pResBlock);

3289
  ASSERT(numOfCols > 0);
5
54liuyao 已提交
3290
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
3291

3292
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3293 3294
  pInfo->pChildren = NULL;
  if (numOfChild > 0) {
3295
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
5
54liuyao 已提交
3296 3297 3298
    for (int32_t i = 0; i < numOfChild; i++) {
      SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp) {
3299 3300
        SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
        pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3301 3302 3303 3304 3305 3306
        taosArrayPush(pInfo->pChildren, &pChildOp);
        continue;
      }
      goto _error;
    }
  }
3307
  pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
3308
  blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
5
54liuyao 已提交
3309 3310
  pInfo->returnUpdate = false;

3311
  pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
5
54liuyao 已提交
3312

5
54liuyao 已提交
3313 3314 3315 3316
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
    pInfo->isFinal = true;
    pOperator->name = "StreamFinalIntervalOperator";
  } else {
5
54liuyao 已提交
3317
    // semi interval operator does not catch result
5
54liuyao 已提交
3318 3319
    pInfo->isFinal = false;
    pOperator->name = "StreamSemiIntervalOperator";
H
Haojun Liao 已提交
3320
    ASSERT(pInfo->aggSup.currentPageId == -1);
5
54liuyao 已提交
3321 3322
  }

5
54liuyao 已提交
3323
  if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
5
54liuyao 已提交
3324 3325
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }
5
54liuyao 已提交
3326 3327 3328 3329
  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
  pInfo->pullIndex = 0;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
3330
  pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
5
54liuyao 已提交
3331
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
3332
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3333
  pInfo->delIndex = 0;
H
Haojun Liao 已提交
3334
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
5
54liuyao 已提交
3335
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
5
54liuyao 已提交
3336

5
54liuyao 已提交
3337
  pOperator->operatorType = pPhyNode->type;
5
54liuyao 已提交
3338 3339 3340 3341
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;

S
shenglian zhou 已提交
3342 3343 3344
  pOperator->fpSet =
      createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
3345
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
3346
    initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
3347
  }
5
54liuyao 已提交
3348 3349 3350 3351 3352 3353 3354 3355
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
3356
  destroyStreamFinalIntervalOperatorInfo(pInfo);
5
54liuyao 已提交
3357 3358 3359
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
5
54liuyao 已提交
3360
}
5
54liuyao 已提交
3361 3362 3363

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
3364
  void** pIte = NULL;
3365
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
3366
    SArray* pWins = (SArray*)(*pIte);
3367 3368 3369
    taosArrayDestroy(pWins);
  }
  taosHashCleanup(pSup->pResultRows);
5
54liuyao 已提交
3370
  destroyDiskbasedBuf(pSup->pResultBuf);
3371
  blockDataDestroy(pSup->pScanBlock);
5
54liuyao 已提交
3372 3373
}

5
54liuyao 已提交
3374 3375 3376 3377
void destroyStateWinInfo(void* ptr) {
  if (ptr == NULL) {
    return;
  }
3378
  SStateWindowInfo* pWin = (SStateWindowInfo*)ptr;
5
54liuyao 已提交
3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393
  taosMemoryFreeClear(pWin->stateKey.pData);
}

void destroyStateStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
  void** pIte = NULL;
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
    SArray* pWins = (SArray*)(*pIte);
    taosArrayDestroyEx(pWins, (FDelete)destroyStateWinInfo);
  }
  taosHashCleanup(pSup->pResultRows);
  destroyDiskbasedBuf(pSup->pResultBuf);
  blockDataDestroy(pSup->pScanBlock);
}

3394
void destroyStreamSessionAggOperatorInfo(void* param) {
5
54liuyao 已提交
3395
  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;
3396
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
3397 3398
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
3399 3400 3401
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
3402
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
3403
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
3404
      destroyStreamSessionAggOperatorInfo(pChInfo);
3405 3406 3407
      taosMemoryFreeClear(pChild);
    }
  }
5
54liuyao 已提交
3408 3409 3410 3411
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  blockDataDestroy(pInfo->pWinBlock);
  blockDataDestroy(pInfo->pUpdateRes);
5
54liuyao 已提交
3412 3413
  destroySqlFunctionCtx(pInfo->pDummyCtx, 0);
  taosHashCleanup(pInfo->pStDeleted);
3414

D
dapan1121 已提交
3415
  taosMemoryFreeClear(param);
5
54liuyao 已提交
3416 3417
}

3418 3419
int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,
                        SSDataBlock* pResultBlock) {
3420 3421 3422 3423
  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
  if (code != TSDB_CODE_SUCCESS) {
    return code;
  }
H
Haojun Liao 已提交
3424
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
3425

3426
  initBasicInfo(pBasicInfo, pResultBlock);
3427

5
54liuyao 已提交
3428
  for (int32_t i = 0; i < numOfCols; ++i) {
3429
    pSup->pCtx[i].saveHandle.pBuf = NULL;
5
54liuyao 已提交
3430
  }
3431

3432
  ASSERT(numOfCols > 0);
5
54liuyao 已提交
3433 3434 3435 3436 3437 3438 3439 3440
  return TSDB_CODE_SUCCESS;
}

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {
  for (int i = 0; i < nums; i++) {
    pDummy[i].functionId = pCtx[i].functionId;
  }
}
5
54liuyao 已提交
3441

X
Xiaoyu Wang 已提交
3442
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
3443 3444 3445 3446 3447 3448 3449 3450 3451 3452
                    uint16_t type, int32_t tsColIndex) {
  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
    pScanInfo->tsColIndex = tsColIndex;
  }

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
    initDownStream(downstream->pDownstream[0], pAggSup, gap, waterMark, type, tsColIndex);
    return;
  }
3453
  SStreamScanInfo* pScanInfo = downstream->info;
3454
  pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
5
54liuyao 已提交
3455
  pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
5
54liuyao 已提交
3456 3457
}

3458 3459
int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx,
                                int32_t numOfOutput) {
3460 3461 3462
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo));
}

3463 3464 3465 3466 3467 3468 3469
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                  SExecTaskInfo* pTaskInfo) {
  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
  int32_t                        numOfCols = 0;
  SExprInfo*                     pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock*                   pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;
X
Xiaoyu Wang 已提交
3470
  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
3471
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3472 3473 3474 3475
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

3476 3477
  pOperator->pTaskInfo = pTaskInfo;

3478
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3479 3480 3481
  if (pSessionNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar);
3482
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
5
54liuyao 已提交
3483 3484 3485 3486
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
3487
  SExprSupp* pSup = &pOperator->exprSupp;
5
54liuyao 已提交
3488

3489
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
3490 3491 3492
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
3493

3494
  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
3495 3496 3497
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
X
Xiaoyu Wang 已提交
3498

5
54liuyao 已提交
3499 3500 3501 3502
  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }
3503
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
3504

3505 3506
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType, .maxTs = INT64_MIN};
H
Haojun Liao 已提交
3507 3508

  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3509 3510
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

5
54liuyao 已提交
3511 3512 3513 3514
  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  if (pSessionNode->window.pTsEnd) {
    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;
  }
3515
  pInfo->gap = pSessionNode->gap;
5
54liuyao 已提交
3516 3517 3518 3519 3520
  pInfo->binfo.pRes = pResBlock;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
5
54liuyao 已提交
3521
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3522
  pInfo->pChildren = NULL;
5
54liuyao 已提交
3523 3524
  pInfo->isFinal = false;
  pInfo->pPhyNode = pPhyNode;
5
54liuyao 已提交
3525
  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
5
54liuyao 已提交
3526 3527

  pOperator->name = "StreamSessionWindowAggOperator";
3528
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
5
54liuyao 已提交
3529 3530 3531
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
3532 3533 3534
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
3535
  if (downstream) {
L
Liu Jicong 已提交
3536 3537
    initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType,
                   pInfo->primaryTsIndex);
5
54liuyao 已提交
3538 3539
    code = appendDownstream(pOperator, &downstream, 1);
  }
5
54liuyao 已提交
3540 3541 3542 3543
  return pOperator;

_error:
  if (pInfo != NULL) {
3544
    destroyStreamSessionAggOperatorInfo(pInfo);
5
54liuyao 已提交
3545 3546 3547 3548 3549 3550 3551 3552
  }

  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

int64_t getSessionWindowEndkey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
3553
  SArray*            pWinInfos = (SArray*)data;
5
54liuyao 已提交
3554 3555 3556
  SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index);
  return pWin->win.ekey;
}
5
54liuyao 已提交
3557 3558

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {
5
54liuyao 已提交
3559
  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {
5
54liuyao 已提交
3560 3561 3562 3563 3564
    return true;
  }
  return false;
}

3565
bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); }
5
54liuyao 已提交
3566

3567
static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs, int32_t index) {
L
Liu Jicong 已提交
3568 3569
  SResultWindowInfo win = {
      .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
5
54liuyao 已提交
3570 3571 3572
  return taosArrayInsert(pWinInfos, index, &win);
}

3573
static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs) {
L
Liu Jicong 已提交
3574 3575
  SResultWindowInfo win = {
      .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
5
54liuyao 已提交
3576 3577 3578
  return taosArrayPush(pWinInfos, &win);
}

3579
SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) {
3580
  void**  ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t));
3581 3582 3583
  SArray* pWinInfos = NULL;
  if (ite == NULL) {
    pWinInfos = taosArrayInit(1024, pAggSup->valueSize);
3584
    taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void*));
3585 3586 3587 3588 3589 3590
  } else {
    pWinInfos = *ite;
  }
  return pWinInfos;
}

5
54liuyao 已提交
3591 3592
// don't add new window
SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
3593
                                       int64_t gap, int32_t* pIndex) {
5
54liuyao 已提交
3594 3595 3596 3597 3598 3599 3600 3601
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    return NULL;
  }
  // find the first position which is smaller than the key
3602
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isInWindow(pWin, startTs, gap)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isInWindow(pWin, startTs, gap)) {
      *pIndex = index + 1;
      return pWin;
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
3618
      *pIndex = index + 1;
5
54liuyao 已提交
3619 3620 3621 3622 3623 3624 3625
      return pWin;
    }
  }

  return NULL;
}

3626 3627
SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
                                        int64_t gap, int32_t* pIndex) {
3628 3629 3630
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

5
54liuyao 已提交
3631 3632
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
5
54liuyao 已提交
3633
    *pIndex = 0;
3634
    return addNewSessionWindow(pWinInfos, startTs, endTs);
5
54liuyao 已提交
3635 3636
  }
  // find the first position which is smaller than the key
5
54liuyao 已提交
3637
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3638 3639 3640
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
5
54liuyao 已提交
3641
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3642 3643 3644 3645 3646 3647 3648
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
5
54liuyao 已提交
3649
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3650 3651
      *pIndex = index + 1;
      return pWin;
5
54liuyao 已提交
3652 3653 3654
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
      *pIndex = index;
      return pWin;
5
54liuyao 已提交
3655 3656 3657 3658 3659
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
3660
    return addNewSessionWindow(pWinInfos, startTs, endTs);
5
54liuyao 已提交
3661
  }
5
54liuyao 已提交
3662
  *pIndex = index + 1;
3663
  return insertNewSessionWindow(pWinInfos, startTs, endTs, index + 1);
5
54liuyao 已提交
3664 3665
}

dengyihao's avatar
dengyihao 已提交
3666 3667
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId,
                                int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted) {
5
54liuyao 已提交
3668
  for (int32_t i = start; i < rows; ++i) {
3669
    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {
5
54liuyao 已提交
3670 3671
      return i - start;
    }
5
54liuyao 已提交
3672
    if (pWinInfo->win.skey > pStartTs[i]) {
5
54liuyao 已提交
3673
      if (pStDeleted && pWinInfo->isOutput) {
H
Haojun Liao 已提交
3674
        SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
3675
        taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
3676 3677
        pWinInfo->isOutput = false;
      }
5
54liuyao 已提交
3678 3679 3680 3681 3682
      pWinInfo->win.skey = pStartTs[i];
    }
    pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pStartTs[i]);
    if (pEndTs) {
      pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pEndTs[i]);
5
54liuyao 已提交
3683 3684 3685 3686 3687
    }
  }
  return rows - start;
}

X
Xiaoyu Wang 已提交
3688
static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,
3689
                                  uint64_t groupId, int32_t numOfOutput, int32_t* rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3690
                                  SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) {
5
54liuyao 已提交
3691 3692
  assert(pWinInfo->win.skey <= pWinInfo->win.ekey);
  // too many time window in query
3693
  int32_t size = taosArrayGetSize(pAggSup->pCurWins);
3694
  if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH && size > MAX_INTERVAL_TIME_WINDOW) {
3695
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
5
54liuyao 已提交
3696
  }
X
Xiaoyu Wang 已提交
3697

5
54liuyao 已提交
3698
  if (pWinInfo->pos.pageId == -1) {
3699
    *pResult = getNewResultRow(pAggSup->pResultBuf, &pAggSup->currentPageId, pAggSup->resultRowSize);
5
54liuyao 已提交
3700 3701 3702 3703 3704 3705 3706 3707
    if (*pResult == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }

    // add a new result set for a new group
    pWinInfo->pos.pageId = (*pResult)->pageId;
    pWinInfo->pos.offset = (*pResult)->offset;
  } else {
3708
    *pResult = getResultRowByPos(pAggSup->pResultBuf, &pWinInfo->pos, true);
5
54liuyao 已提交
3709 3710 3711 3712 3713 3714 3715 3716
    if (!(*pResult)) {
      qError("getResultRowByPos return NULL, TID:%s", GET_TASKID(pTaskInfo));
      return TSDB_CODE_FAILED;
    }
  }

  // set time window for current result
  (*pResult)->win = pWinInfo->win;
3717
  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
5
54liuyao 已提交
3718 3719 3720
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3721 3722 3723
static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup,
                                  SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock,
                                  SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3724
                                  int32_t numOutput, SOperatorInfo* pOperator) {
3725
  SExprSupp*     pSup = &pOperator->exprSupp;
3726 3727
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

X
Xiaoyu Wang 已提交
3728 3729
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId);
  TSKEY*           tsCols = (int64_t*)pColDataInfo->pData;
3730 3731
  int32_t          code = setWindowOutputBuf(pCurWin, pResult, pSup->pCtx, pSDataBlock->info.groupId, numOutput,
                                             pSup->rowEntryInfoOffset, pAggSup, pTaskInfo);
5
54liuyao 已提交
3732 3733 3734
  if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }
5
54liuyao 已提交
3735
  updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false);
3736
  doApplyFunctions(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, pSDataBlock->info.rows, numOutput);
5
54liuyao 已提交
3737 3738 3739
  SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pCurWin->pos.pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pAggSup->pResultBuf, bufPage);
5
54liuyao 已提交
3740 3741 3742
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3743 3744
static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                              SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3745
                              int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3746
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3747
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3748 3749
}

X
Xiaoyu Wang 已提交
3750 3751
static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                                   SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex,
3752
                                   int32_t winRows, int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3753
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3754
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3755 3756
}

5
54liuyao 已提交
3757 3758
int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) {
  SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex);
X
Xiaoyu Wang 已提交
3759
  int32_t            size = taosArrayGetSize(pWinInfos);
5
54liuyao 已提交
3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i);
    if (!isInWindow(pCurWin, pWinInfo->win.skey, gap)) {
      return i - startIndex - 1;
    }
  }

  return size - startIndex - 1;
}

5
54liuyao 已提交
3771
void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId,
3772
                       int32_t numOfOutput, SHashObj* pStUpdated, SHashObj* pStDeleted, SOperatorInfo* pOperator) {
3773
  SExprSupp*     pSup = &pOperator->exprSupp;
3774 3775
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

3776
  SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex);
X
Xiaoyu Wang 已提交
3777
  SResultRow*        pCurResult = NULL;
3778
  setWindowOutputBuf(pCurWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3779
                     &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3780
  num += startIndex + 1;
3781
  ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins));
5
54liuyao 已提交
3782 3783
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < num; i++) {
3784
    SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i);
X
Xiaoyu Wang 已提交
3785
    SResultRow*        pWinResult = NULL;
3786
    setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3787
                       &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3788
    pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey);
3789 3790
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true);
    compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
3791 3792
    taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition));
    if (pWinInfo->isOutput) {
H
Haojun Liao 已提交
3793
      SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
3794
      taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
3795 3796
      pWinInfo->isOutput = false;
    }
3797
    taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
5
54liuyao 已提交
3798 3799
    SFilePage* tmpPage = getBufPage(pInfo->streamAggSup.pResultBuf, pWinInfo->pos.pageId);
    releaseBufPage(pInfo->streamAggSup.pResultBuf, tmpPage);
5
54liuyao 已提交
3800
  }
5
54liuyao 已提交
3801 3802 3803 3804
  SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pCurWin->pos.pageId);
  ASSERT(num > 0);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
5
54liuyao 已提交
3805 3806
}

3807 3808
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
                                   SHashObj* pStDeleted, bool hasEndTs) {
X
Xiaoyu Wang 已提交
3809
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3810
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3811
  bool                           masterScan = true;
3812
  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
5
54liuyao 已提交
3813
  uint64_t                       groupId = pSDataBlock->info.groupId;
X
Xiaoyu Wang 已提交
3814 3815 3816 3817 3818
  int64_t                        gap = pInfo->gap;
  int64_t                        code = TSDB_CODE_SUCCESS;

  int32_t     step = 1;
  bool        ascScan = true;
5
54liuyao 已提交
3819 3820
  TSKEY*      startTsCols = NULL;
  TSKEY*      endTsCols = NULL;
5
54liuyao 已提交
3821
  SResultRow* pResult = NULL;
X
Xiaoyu Wang 已提交
3822
  int32_t     winRows = 0;
5
54liuyao 已提交
3823

5
54liuyao 已提交
3824 3825 3826 3827 3828 3829
  ASSERT(pSDataBlock->pDataBlock);
  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  startTsCols = (int64_t*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = NULL;
  if (hasEndTs) {
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);
5
54liuyao 已提交
3830
  } else {
5
54liuyao 已提交
3831
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
3832
  }
5
54liuyao 已提交
3833
  endTsCols = (int64_t*)pEndTsCol->pData;
X
Xiaoyu Wang 已提交
3834

5
54liuyao 已提交
3835
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
3836
  for (int32_t i = 0; i < pSDataBlock->info.rows;) {
5
54liuyao 已提交
3837
    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
3838 3839 3840
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
3841
    int32_t            winIndex = 0;
3842
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex);
dengyihao's avatar
dengyihao 已提交
3843 3844
    winRows = updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, groupId, pSDataBlock->info.rows, i, pInfo->gap,
                                      pStDeleted);
3845
    code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
3846
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
3847
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3848
    }
5
54liuyao 已提交
3849

3850
    int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap);
5
54liuyao 已提交
3851
    if (winNum > 0) {
3852
      compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, pStDeleted, pOperator);
5
54liuyao 已提交
3853
    }
5
54liuyao 已提交
3854
    pCurWin->isClosed = false;
5
54liuyao 已提交
3855
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
H
Haojun Liao 已提交
3856 3857
      SWinKey value = {.ts = pCurWin->win.skey, .groupId = groupId};
      code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
5
54liuyao 已提交
3858
      if (code != TSDB_CODE_SUCCESS) {
3859
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3860 3861
      }
      pCurWin->isOutput = true;
5
54liuyao 已提交
3862 3863 3864 3865 3866
    }
    i += winRows;
  }
}

5
54liuyao 已提交
3867
void deleteWindow(SArray* pWinInfos, int32_t index, FDelete fp) {
5
54liuyao 已提交
3868
  ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos));
5
54liuyao 已提交
3869 3870 3871 3872
  if (fp) {
    void* ptr = taosArrayGet(pWinInfos, index);
    fp(ptr);
  }
5
54liuyao 已提交
3873 3874 3875
  taosArrayRemove(pWinInfos, index);
}

3876 3877
static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result,
                                FDelete fp) {
5
54liuyao 已提交
3878
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
3879
  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;
5
54liuyao 已提交
3880
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
3881
  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;
3882
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
3883
  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;
5
54liuyao 已提交
3884 3885
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    int32_t winIndex = 0;
3886 3887
    while (1) {
      SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex);
5
54liuyao 已提交
3888 3889 3890
      if (!pCurWin) {
        break;
      }
3891
      SResultWindowInfo delWin = *pCurWin;
5
54liuyao 已提交
3892
      deleteWindow(pAggSup->pCurWins, winIndex, fp);
5
54liuyao 已提交
3893
      if (result) {
3894 3895
        delWin.groupId = gpDatas[i];
        taosArrayPush(result, &delWin);
5
54liuyao 已提交
3896 3897 3898 3899 3900
      }
    }
  }
}

3901 3902
static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock, int32_t tsIndex,
                                  int32_t numOfOutput, int64_t gap, SArray* result) {
5
54liuyao 已提交
3903
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
3904
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
H
Haojun Liao 已提交
3905 3906
  SColumnInfoData* pGpDataInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*        gpCols = (uint64_t*)pGpDataInfo->pData;
X
Xiaoyu Wang 已提交
3907
  int32_t          step = 0;
5
54liuyao 已提交
3908
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
3909
    int32_t            winIndex = 0;
H
Haojun Liao 已提交
3910
    SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, tsCols[i], INT64_MIN, gpCols[i], gap, &winIndex);
5
54liuyao 已提交
3911
    if (!pCurWin || pCurWin->pos.pageId == -1) {
5
54liuyao 已提交
3912
      // window has been closed.
5
54liuyao 已提交
3913
      step = 1;
5
54liuyao 已提交
3914 3915
      continue;
    }
5
54liuyao 已提交
3916
    step = updateSessionWindowInfo(pCurWin, tsCols, NULL, 0, pBlock->info.rows, i, gap, NULL);
5
54liuyao 已提交
3917
    ASSERT(isInWindow(pCurWin, tsCols[i], gap));
3918
    doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
3919
    if (result) {
3920
      pCurWin->groupId = gpCols[i];
3921 3922
      taosArrayPush(result, pCurWin);
    }
5
54liuyao 已提交
3923 3924 3925
  }
}

5
54liuyao 已提交
3926
static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) {
X
Xiaoyu Wang 已提交
3927
  void*  pData = NULL;
5
54liuyao 已提交
3928
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
3929
  while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) {
5
54liuyao 已提交
3930 3931 3932 3933 3934 3935
    void* key = taosHashGetKey(pData, &keyLen);
    ASSERT(keyLen == sizeof(SResultRowPosition));
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    if (pos == NULL) {
      return TSDB_CODE_QRY_OUT_OF_MEMORY;
    }
H
Haojun Liao 已提交
3936
    pos->groupId = ((SWinKey*)pData)->groupId;
5
54liuyao 已提交
3937
    pos->pos = *(SResultRowPosition*)key;
H
Haojun Liao 已提交
3938
    *(int64_t*)pos->key = ((SWinKey*)pData)->ts;
5
54liuyao 已提交
3939 3940
    taosArrayPush(pUpdated, &pos);
  }
5
54liuyao 已提交
3941
  taosArraySort(pUpdated, resultrowComparAsc);
5
54liuyao 已提交
3942 3943 3944 3945 3946
  return TSDB_CODE_SUCCESS;
}

void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
  blockDataCleanup(pBlock);
3947 3948 3949 3950 3951
  int32_t size = taosHashGetSize(pStDeleted);
  if (size == 0) {
    return;
  }
  blockDataEnsureCapacity(pBlock, size);
5
54liuyao 已提交
3952
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
3953
  while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
H
Haojun Liao 已提交
3954
    SWinKey*         res = *Ite;
3955 3956 3957 3958 3959 3960
    SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
    colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->ts, false);
    SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
    colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->ts, false);
    SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
    colDataAppendNULL(pUidCol, pBlock->info.rows);
5
54liuyao 已提交
3961 3962
    SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
    colDataAppend(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false);
3963 3964 3965 3966
    SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
    colDataAppendNULL(pCalStCol, pBlock->info.rows);
    SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
    colDataAppendNULL(pCalEdCol, pBlock->info.rows);
5
54liuyao 已提交
3967 3968 3969 3970 3971 3972 3973 3974 3975 3976
    pBlock->info.rows += 1;
    if (pBlock->info.rows + 1 >= pBlock->info.capacity) {
      break;
    }
  }
  if ((*Ite) == NULL) {
    taosHashClear(pStDeleted);
  }
}

L
Liu Jicong 已提交
3977 3978
static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t numOfOutput,
                              SOperatorInfo* pOperator, SHashObj* pStUpdated, bool needCreate) {
3979
  SExprSupp*     pSup = &pOperator->exprSupp;
3980 3981
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

3982 3983
  int32_t size = taosArrayGetSize(pWinArray);
  ASSERT(pInfo->pChildren);
3984

3985 3986
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i);
X
Xiaoyu Wang 已提交
3987
    SResultRow*        pCurResult = NULL;
3988 3989 3990
    uint64_t           groupId = pParentWin->groupId;
    int32_t            winIndex = 0;
    if (needCreate) {
L
Liu Jicong 已提交
3991 3992
      pParentWin =
          getSessionTimeWindow(&pInfo->streamAggSup, pParentWin->win.skey, pParentWin->win.ekey, groupId, 0, &winIndex);
3993
    }
3994
    setWindowOutputBuf(pParentWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3995
                       &pInfo->streamAggSup, pTaskInfo);
3996
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
3997
    int32_t num = 0;
3998
    for (int32_t j = 0; j < numOfChildren; j++) {
X
Xiaoyu Wang 已提交
3999
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
4000
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
4001
      SArray*                        pChWins = getWinInfos(&pChInfo->streamAggSup, groupId);
X
Xiaoyu Wang 已提交
4002 4003
      int32_t                        chWinSize = taosArrayGetSize(pChWins);
      int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
4004
      if (index < 0) {
4005
        index = 0;
5
54liuyao 已提交
4006 4007
      }
      for (int32_t k = index; k < chWinSize; k++) {
5
54liuyao 已提交
4008 4009
        SResultWindowInfo* pChWin = taosArrayGet(pChWins, k);
        if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) {
4010
          SResultRow* pChResult = NULL;
5
54liuyao 已提交
4011
          setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
4012
                             pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
4013 4014
          updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pChWin->win, true);
          compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
4015 4016
          SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
          releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage);
4017
          num++;
4018
          continue;
5
54liuyao 已提交
4019 4020
        } else if (!pChWin->isClosed) {
          break;
4021 4022 4023
        }
      }
    }
4024 4025 4026 4027 4028 4029 4030
    if (num == 0 && needCreate) {
      deleteWindow(pInfo->streamAggSup.pCurWins, winIndex, NULL);
    }
    if (pStUpdated && num > 0) {
      SWinKey value = {.ts = pParentWin->win.skey, .groupId = groupId};
      taosHashPut(pStUpdated, &pParentWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
    }
5
54liuyao 已提交
4031 4032 4033 4034
    SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId);
    ASSERT(size > 0);
    setBufPageDirty(bufPage, true);
    releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
4035 4036 4037
  }
}

X
Xiaoyu Wang 已提交
4038
typedef SResultWindowInfo* (*__get_win_info_)(void*);
5
54liuyao 已提交
4039 4040
SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
5
54liuyao 已提交
4041

4042
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn,
5
54liuyao 已提交
4043
                           bool delete, FDelete fp) {
5
54liuyao 已提交
4044
  // Todo(liuyao) save window to tdb
4045
  void** pIte = NULL;
5
54liuyao 已提交
4046
  size_t keyLen = 0;
4047
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
5
54liuyao 已提交
4048
    uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen);
4049 4050
    SArray*   pWins = (SArray*)(*pIte);
    int32_t   size = taosArrayGetSize(pWins);
4051 4052 4053
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
5
54liuyao 已提交
4054
      if (isCloseWindow(&pSeWin->win, pTwSup)) {
4055 4056
        if (!pSeWin->isClosed) {
          pSeWin->isClosed = true;
5
54liuyao 已提交
4057
          if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {
5
54liuyao 已提交
4058
            int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed);
5
54liuyao 已提交
4059 4060 4061
            if (code != TSDB_CODE_SUCCESS) {
              return code;
            }
4062 4063
            pSeWin->isOutput = true;
          }
5
54liuyao 已提交
4064
          if (delete) {
5
54liuyao 已提交
4065
            deleteWindow(pWins, i, fp);
5
54liuyao 已提交
4066 4067 4068
            i--;
            size = taosArrayGetSize(pWins);
          }
5
54liuyao 已提交
4069
        }
4070
        continue;
5
54liuyao 已提交
4071
      }
4072
      break;
5
54liuyao 已提交
4073 4074 4075 4076 4077
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
4078
static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs, bool delete, FDelete fp) {
5
54liuyao 已提交
4079 4080 4081 4082 4083
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);
    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4084
    closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL, getResWinForSession, delete, fp);
5
54liuyao 已提交
4085 4086 4087
  }
}

4088
int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) {
4089
  void** pIte = NULL;
4090
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
4091
    SArray* pWins = (SArray*)(*pIte);
4092 4093 4094 4095 4096 4097 4098 4099
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (!pSeWin->isClosed) {
        int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed);
        pSeWin->isOutput = true;
      }
5
54liuyao 已提交
4100 4101 4102 4103 4104
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
4105 4106 4107 4108
static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) {
  int32_t size = taosArrayGetSize(pResWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i);
H
Haojun Liao 已提交
4109
    SWinKey            res = {.ts = pWinInfo->win.skey, .groupId = pWinInfo->groupId};
4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122
    taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
  }
}

static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) {
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWin = taosArrayGet(pWins, i);
    taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition));
  }
}

int32_t compareWinKey(void* pKey, void* data, int32_t index) {
L
Liu Jicong 已提交
4123
  SArray*     res = (SArray*)data;
4124
  SResKeyPos* pos = taosArrayGetP(res, index);
L
Liu Jicong 已提交
4125
  SWinKey*    pData = (SWinKey*)pKey;
4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147
  if (pData->ts == *(int64_t*)pos->key) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->ts > *(int64_t*)pos->key) {
    return 1;
  }
  return -1;
}

static void removeSessionDeleteResults(SArray* update, SHashObj* pStDeleted) {
  int32_t size = taosHashGetSize(pStDeleted);
  if (size == 0) {
    return;
  }

  int32_t num = taosArrayGetSize(update);
  for (int32_t i = 0; i < num; i++) {
    SResKeyPos* pos = taosArrayGetP(update, i);
L
Liu Jicong 已提交
4148
    SWinKey     winKey = {.ts = *(int64_t*)pos->key, .groupId = pos->groupId};
4149
    taosHashRemove(pStDeleted, &winKey, sizeof(SWinKey));
5
54liuyao 已提交
4150 4151 4152
  }
}

5
54liuyao 已提交
4153
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
5
54liuyao 已提交
4154
  SExprSupp*                     pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4155
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4156
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
5
54liuyao 已提交
4157 4158 4159 4160
  TSKEY                          maxTs = INT64_MIN;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
4161 4162
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4163
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4164 4165
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
4166
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
4167
    if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
5
54liuyao 已提交
4168 4169
      doSetOperatorCompleted(pOperator);
    }
5
54liuyao 已提交
4170
    printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4171 4172 4173
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
4174 4175
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
4176
  SOperatorInfo* downstream = pOperator->pDownstream[0];
L
Liu Jicong 已提交
4177
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);  // SResKeyPos
5
54liuyao 已提交
4178 4179 4180 4181 4182
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4183
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv");
4184

5
54liuyao 已提交
4185
    if (pBlock->info.type == STREAM_CLEAR) {
X
Xiaoyu Wang 已提交
4186
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
H
Haojun Liao 已提交
4187 4188
      doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, START_TS_COLUMN_INDEX,
                            pOperator->exprSupp.numOfExprs, 0, pWins);
5
54liuyao 已提交
4189 4190
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
X
Xiaoyu Wang 已提交
4191
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
4192
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
L
Liu Jicong 已提交
4193 4194
        doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX,
                              pChildOp->exprSupp.numOfExprs, 0, NULL);
4195
        rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, NULL, false);
4196 4197
      }
      taosArrayDestroy(pWins);
5
54liuyao 已提交
4198
      continue;
5
54liuyao 已提交
4199 4200 4201
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      // gap must be 0
5
54liuyao 已提交
4202
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL);
5
54liuyao 已提交
4203 4204 4205 4206 4207
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
        // gap must be 0
5
54liuyao 已提交
4208
        doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL);
4209
        rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated, true);
5
54liuyao 已提交
4210 4211
      }
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
4212
      removeSessionResults(pStUpdated, pWins);
5
54liuyao 已提交
4213 4214
      taosArrayDestroy(pWins);
      continue;
4215
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4216
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
5
54liuyao 已提交
4217
      continue;
5
54liuyao 已提交
4218
    }
5
54liuyao 已提交
4219

5
54liuyao 已提交
4220 4221 4222 4223
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4224
    // the pDataBlock are always the same one, no need to call this again
4225
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4226 4227 4228 4229 4230 4231
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo));
    if (IS_FINAL_OP(pInfo)) {
      int32_t chIndex = getChildIndex(pBlock);
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
4232 4233
        SOperatorInfo* pChildOp =
            createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
5
54liuyao 已提交
4234
        if (!pChildOp) {
4235
          T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4236 4237 4238
        }
        taosArrayPush(pInfo->pChildren, &pChildOp);
      }
4239
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
4240 4241
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true);
4242
    }
5
54liuyao 已提交
4243
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
4244
    maxTs = TMAX(maxTs, pBlock->info.watermark);
5
54liuyao 已提交
4245
  }
5
54liuyao 已提交
4246 4247

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4248 4249
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
4250

4251
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession,
5
54liuyao 已提交
4252 4253
                     pInfo->ignoreExpiredData, NULL);
  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, NULL);
5
54liuyao 已提交
4254
  copyUpdateResult(pStUpdated, pUpdated);
4255
  removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
5
54liuyao 已提交
4256 4257
  taosHashCleanup(pStUpdated);

4258
  finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4259 4260 4261 4262
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4263
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4264 4265 4266
    return pInfo->pDelRes;
  }
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
4267
  printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4268 4269 4270 4271
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
4272
  void** pIte = NULL;
5
54liuyao 已提交
4273
  while ((pIte = taosHashIterate(pInfo->streamAggSup.pResultRows, pIte)) != NULL) {
4274
    SArray* pWins = (SArray*)(*pIte);
5
54liuyao 已提交
4275 4276 4277 4278 4279 4280 4281 4282
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      SResultWindowInfo* pWin = (SResultWindowInfo*)taosArrayGet(pWins, i);
      pWin->pos.pageId = -1;
      pWin->pos.offset = -1;
    }
  }
  clearDiskbasedBuf(pInfo->streamAggSup.pResultBuf);
4283
  pInfo->streamAggSup.currentPageId = -1;
5
54liuyao 已提交
4284 4285 4286 4287 4288 4289 4290
}

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
  TSKEY                          maxTs = INT64_MIN;
  SExprSupp*                     pSup = &pOperator->exprSupp;
4291

5
54liuyao 已提交
4292 4293
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
4294
  }
L
Liu Jicong 已提交
4295

4296
  {
5
54liuyao 已提交
4297 4298
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
    if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
4299
      printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
4300 4301 4302
      return pBInfo->pRes;
    }

4303 4304
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
H
Haojun Liao 已提交
4305
      printDataBlock(pInfo->pDelRes, "semi session");
5
54liuyao 已提交
4306 4307
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
4308

4309 4310
    if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
      pInfo->returnUpdate = false;
5
54liuyao 已提交
4311
      // process the rest of the data
H
Haojun Liao 已提交
4312
      printDataBlock(pInfo->pUpdateRes, "semi session");
5
54liuyao 已提交
4313 4314
      return pInfo->pUpdateRes;
    }
4315 4316

    if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
4317
      clearFunctionContext(&pOperator->exprSupp);
4318 4319 4320 4321 4322
      // semi interval operator clear disk buffer
      clearStreamSessionOperator(pInfo);
      pOperator->status = OP_EXEC_DONE;
      return NULL;
    }
5
54liuyao 已提交
4323 4324 4325 4326 4327 4328 4329 4330 4331
  }

  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  SOperatorInfo* downstream = pOperator->pDownstream[0];
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
4332
      clearSpecialDataBlock(pInfo->pUpdateRes);
4333
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
4334 4335
      break;
    }
H
Haojun Liao 已提交
4336
    printDataBlock(pBlock, "semi session recv");
5
54liuyao 已提交
4337

5
54liuyao 已提交
4338
    if (pBlock->info.type == STREAM_CLEAR) {
5
54liuyao 已提交
4339
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
H
Haojun Liao 已提交
4340
      doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, START_TS_COLUMN_INDEX, pSup->numOfExprs, 0, pWins);
5
54liuyao 已提交
4341 4342
      removeSessionResults(pStUpdated, pWins);
      taosArrayDestroy(pWins);
H
Haojun Liao 已提交
4343
      copyDataBlock(pInfo->pUpdateRes, pBlock);
4344
      pInfo->returnUpdate = true;
5
54liuyao 已提交
4345
      break;
5
54liuyao 已提交
4346 4347
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      // gap must be 0
4348 4349 4350 4351 4352
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL);
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
      removeSessionResults(pStUpdated, pWins);
      taosArrayDestroy(pWins);
5
54liuyao 已提交
4353
      break;
5
54liuyao 已提交
4354 4355 4356 4357 4358
    } else if (pBlock->info.type == STREAM_GET_ALL) {
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
      continue;
    }

5
54liuyao 已提交
4359 4360 4361 4362
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
5
54liuyao 已提交
4363 4364
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
4365
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, NULL, false);
5
54liuyao 已提交
4366 4367 4368 4369
    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
4370
  pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;
4371

5
54liuyao 已提交
4372
  copyUpdateResult(pStUpdated, pUpdated);
4373
  removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
5
54liuyao 已提交
4374
  taosHashCleanup(pStUpdated);
5
54liuyao 已提交
4375

4376 4377
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4378
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
5
54liuyao 已提交
4379
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
4380 4381 4382

  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
  if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
4383
    printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
4384 4385 4386
    return pBInfo->pRes;
  }

4387 4388
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
H
Haojun Liao 已提交
4389
    printDataBlock(pInfo->pDelRes, "semi session");
5
54liuyao 已提交
4390 4391
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
4392

4393 4394
  if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
    pInfo->returnUpdate = false;
5
54liuyao 已提交
4395
    // process the rest of the data
H
Haojun Liao 已提交
4396
    printDataBlock(pInfo->pUpdateRes, "semi session");
5
54liuyao 已提交
4397 4398
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
4399 4400 4401

  pOperator->status = OP_EXEC_DONE;
  return NULL;
5
54liuyao 已提交
4402
}
4403

4404 4405
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                       SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
4406 4407
  int32_t        code = TSDB_CODE_OUT_OF_MEMORY;
  SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo);
4408 4409 4410
  if (pOperator == NULL) {
    goto _error;
  }
4411
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
5
54liuyao 已提交
4412 4413 4414 4415 4416 4417

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
    pInfo->isFinal = true;
    pOperator->name = "StreamSessionFinalAggOperator";
  } else {
    pInfo->isFinal = false;
H
Haojun Liao 已提交
4418
    pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
4419 4420 4421
    blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
    pOperator->name = "StreamSessionSemiAggOperator";
    pOperator->fpSet =
4422 4423
        createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL,
                            destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
4424 4425 4426 4427 4428
  }
  pOperator->operatorType = pPhyNode->type;
  if (numOfChild > 0) {
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
    for (int32_t i = 0; i < numOfChild; i++) {
4429
      SOperatorInfo* pChild = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
5
54liuyao 已提交
4430 4431 4432 4433
      if (pChild == NULL) {
        goto _error;
      }
      taosArrayPush(pInfo->pChildren, &pChild);
4434 4435 4436 4437 4438 4439
    }
  }
  return pOperator;

_error:
  if (pInfo != NULL) {
4440
    destroyStreamSessionAggOperatorInfo(pInfo);
4441 4442 4443 4444 4445 4446
  }

  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5
54liuyao 已提交
4447

4448
void destroyStreamStateOperatorInfo(void* param) {
X
Xiaoyu Wang 已提交
4449
  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;
4450
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
4451
  destroyStateStreamAggSupporter(&pInfo->streamAggSup);
5
54liuyao 已提交
4452 4453 4454 4455
  cleanupGroupResInfo(&pInfo->groupResInfo);
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
4456
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
5
54liuyao 已提交
4457
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
4458
      destroyStreamSessionAggOperatorInfo(pChInfo);
5
54liuyao 已提交
4459 4460 4461 4462
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
5
54liuyao 已提交
4463 4464 4465 4466
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  taosHashCleanup(pInfo->pSeDeleted);
  destroySqlFunctionCtx(pInfo->pDummyCtx, 0);
D
dapan1121 已提交
4467 4468

  taosMemoryFreeClear(param);
5
54liuyao 已提交
4469 4470 4471 4472 4473 4474 4475
}

int64_t getStateWinTsKey(void* data, int32_t index) {
  SStateWindowInfo* pStateWin = taosArrayGet(data, index);
  return pStateWin->winInfo.win.ekey;
}

X
Xiaoyu Wang 已提交
4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487
SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4488 4489 4490 4491 4492 4493 4494 4495
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayPush(pWinInfos, &win);
}

X
Xiaoyu Wang 已提交
4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507
SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayInsert(pWinInfos, index, &win);
}

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {
  if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) {
    return true;
  }
  return false;
}

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {
  return pKeyData && compareVal(pKeyData, &pWin->stateKey);
}

4527 4528 4529
SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
X
Xiaoyu Wang 已提交
4530 4531
  int32_t           size = taosArrayGetSize(pWinInfos);
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index + 1;
      return pWin;
    }
  }
  *pIndex = 0;
  return NULL;
}

4552 4553
SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
                                 SColumn* pCol, int32_t* pIndex) {
4554 4555
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
5
54liuyao 已提交
4556 4557 4558 4559 4560
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    *pIndex = 0;
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
X
Xiaoyu Wang 已提交
4561
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
  *pIndex = index + 1;
  return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol);
}

H
Haojun Liao 已提交
4595 4596 4597
int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, uint64_t groupId,
                              SColumnInfoData* pKeyCol, int32_t rows, int32_t start, bool* allEqual,
                              SHashObj* pSeDeleted) {
5
54liuyao 已提交
4598 4599 4600 4601 4602
  *allEqual = true;
  SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex);
  for (int32_t i = start; i < rows; ++i) {
    char* pKeyData = colDataGetData(pKeyCol, i);
    if (!isTsInWindow(pWinInfo, pTs[i])) {
X
Xiaoyu Wang 已提交
4603
      if (isEqualStateKey(pWinInfo, pKeyData)) {
5
54liuyao 已提交
4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616
        int32_t size = taosArrayGetSize(pWinInfos);
        if (winIndex + 1 < size) {
          SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1);
          // ts belongs to the next window
          if (pTs[i] >= pNextWin->winInfo.win.skey) {
            return i - start;
          }
        }
      } else {
        return i - start;
      }
    }
    if (pWinInfo->winInfo.win.skey > pTs[i]) {
H
Haojun Liao 已提交
4617 4618
      if (pSeDeleted && pWinInfo->winInfo.isOutput) {
        SWinKey res = {.ts = pWinInfo->winInfo.win.skey, .groupId = groupId};
4619
        taosHashPut(pSeDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631
        pWinInfo->winInfo.isOutput = false;
      }
      pWinInfo->winInfo.win.skey = pTs[i];
    }
    pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]);
    if (!isEqualStateKey(pWinInfo, pKeyData)) {
      *allEqual = false;
    }
  }
  return rows - start;
}

H
Haojun Liao 已提交
4632 4633 4634 4635
static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SHashObj* pSeUpdated,
                                SHashObj* pSeDeleted) {
  SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupColInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
X
Xiaoyu Wang 已提交
4636 4637 4638
  TSKEY*           tsCol = (TSKEY*)pTsColInfo->pData;
  bool             allEqual = false;
  int32_t          step = 1;
H
Haojun Liao 已提交
4639
  uint64_t*        gpCol = (uint64_t*)pGroupColInfo->pData;
5
54liuyao 已提交
4640
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
4641
    int32_t           winIndex = 0;
H
Haojun Liao 已提交
4642
    SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], gpCol[i], &winIndex);
5
54liuyao 已提交
4643 4644 4645
    if (!pCurWin) {
      continue;
    }
H
Haojun Liao 已提交
4646
    updateSessionWindowInfo(&pCurWin->winInfo, tsCol, NULL, 0, pBlock->info.rows, i, 0, NULL);
5
54liuyao 已提交
4647
    taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
5
54liuyao 已提交
4648
    deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
5
54liuyao 已提交
4649 4650 4651
  }
}

X
Xiaoyu Wang 已提交
4652 4653 4654
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated,
                                 SHashObj* pStDeleted) {
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
4655
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4656
  bool                         masterScan = true;
4657
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
X
Xiaoyu Wang 已提交
4658 4659 4660 4661 4662 4663 4664
  int64_t                      groupId = pSDataBlock->info.groupId;
  int64_t                      code = TSDB_CODE_SUCCESS;
  int32_t                      step = 1;
  bool                         ascScan = true;
  TSKEY*                       tsCols = NULL;
  SResultRow*                  pResult = NULL;
  int32_t                      winRows = 0;
5
54liuyao 已提交
4665
  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
4666 4667
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
4668
  } else {
X
Xiaoyu Wang 已提交
4669
    return;
5
54liuyao 已提交
4670
  }
L
Liu Jicong 已提交
4671

5
54liuyao 已提交
4672
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
4673
  blockDataEnsureCapacity(pAggSup->pScanBlock, pSDataBlock->info.rows);
L
Liu Jicong 已提交
4674
  SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
X
Xiaoyu Wang 已提交
4675
  for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
5
54liuyao 已提交
4676
    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
4677 4678 4679
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
4680 4681 4682
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
    bool              allEqual = true;
H
Haojun Liao 已提交
4683 4684 4685
    SStateWindowInfo* pCurWin = getStateWindow(pAggSup, tsCols[i], groupId, pKeyData, &pInfo->stateCol, &winIndex);
    winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, groupId, pKeyColInfo, pSDataBlock->info.rows,
                                    i, &allEqual, pStDeleted);
5
54liuyao 已提交
4686
    if (!allEqual) {
4687
      uint64_t uid = 0;
L
Liu Jicong 已提交
4688
      appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, &uid, &groupId);
5
54liuyao 已提交
4689
      taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
5
54liuyao 已提交
4690
      deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
5
54liuyao 已提交
4691 4692
      continue;
    }
4693
    code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
4694
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
4695
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4696 4697 4698
    }
    pCurWin->winInfo.isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
H
Haojun Liao 已提交
4699 4700
      SWinKey value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId};
      code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
5
54liuyao 已提交
4701
      if (code != TSDB_CODE_SUCCESS) {
4702
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713
      }
      pCurWin->winInfo.isOutput = true;
    }
  }
}

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

4714
  SExprSupp*                   pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4715
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4716
  SOptrBasicInfo*              pBInfo = &pInfo->binfo;
L
Liu Jicong 已提交
4717
  int64_t                      maxTs = INT64_MIN;
5
54liuyao 已提交
4718 4719 4720
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4721
      printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4722 4723
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
4724
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
4725
    if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
5
54liuyao 已提交
4726 4727
      doSetOperatorCompleted(pOperator);
    }
5
54liuyao 已提交
4728
    printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4729 4730 4731
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
4732 4733
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
4734
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
4735
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
4736 4737 4738 4739 4740
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4741
    printDataBlock(pBlock, "single state recv");
4742

5
54liuyao 已提交
4743
    if (pBlock->info.type == STREAM_CLEAR) {
H
Haojun Liao 已提交
4744
      doClearStateWindows(&pInfo->streamAggSup, pBlock, pSeUpdated, pInfo->pSeDeleted);
5
54liuyao 已提交
4745
      continue;
4746 4747
    } else if (pBlock->info.type == STREAM_DELETE_DATA) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
5
54liuyao 已提交
4748
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo);
4749
      copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
4750
      removeSessionResults(pSeUpdated, pWins);
4751 4752
      taosArrayDestroy(pWins);
      continue;
4753
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4754
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState);
5
54liuyao 已提交
4755
      continue;
5
54liuyao 已提交
4756
    }
4757

5
54liuyao 已提交
4758 4759 4760 4761
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4762
    // the pDataBlock are always the same one, no need to call this again
4763
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4764
    doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
4765
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
4766
  }
4767
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4768 4769
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
X
Xiaoyu Wang 已提交
4770

4771
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState,
5
54liuyao 已提交
4772 4773
                     pInfo->ignoreExpiredData, destroyStateWinInfo);
  // closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, destroyStateWinInfo);
5
54liuyao 已提交
4774
  copyUpdateResult(pSeUpdated, pUpdated);
5
54liuyao 已提交
4775 4776
  taosHashCleanup(pSeUpdated);

4777 4778
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4779 4780 4781 4782
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4783
    printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4784 4785
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
4786
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
4787
  printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4788 4789 4790
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

4791 4792 4793 4794
int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo));
}

X
Xiaoyu Wang 已提交
4795 4796 4797 4798 4799 4800
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
  SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
4801
  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
4802

X
Xiaoyu Wang 已提交
4803 4804
  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
4805 4806 4807 4808
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

4809 4810
  SExprSupp* pSup = &pOperator->exprSupp;

X
Xiaoyu Wang 已提交
4811
  int32_t    numOfCols = 0;
5
54liuyao 已提交
4812 4813 4814
  SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
4815
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
4816 4817 4818 4819 4820 4821 4822 4823 4824
  if (pStateNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

4825
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
X
Xiaoyu Wang 已提交
4826 4827
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
5
54liuyao 已提交
4828 4829
      .calTrigger = pStateNode->window.triggerType,
      .maxTs = INT64_MIN,
X
Xiaoyu Wang 已提交
4830
  };
5
54liuyao 已提交
4831
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
4832

4833
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
4834 4835 4836 4837
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

4838
  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
4839 4840 4841 4842 4843 4844 4845 4846 4847
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }

4848
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
4849 4850 4851 4852 4853
  pInfo->primaryTsIndex = tsSlotId;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
H
Haojun Liao 已提交
4854
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
5
54liuyao 已提交
4855
  pInfo->pChildren = NULL;
5
54liuyao 已提交
4856
  pInfo->ignoreExpiredData = pStateNode->window.igExpired;
5
54liuyao 已提交
4857 4858

  pOperator->name = "StreamStateAggOperator";
4859
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
5
54liuyao 已提交
4860 4861 4862 4863
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
4864 4865
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL,
                                         destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
L
Liu Jicong 已提交
4866 4867
  initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType,
                 pInfo->primaryTsIndex);
5
54liuyao 已提交
4868 4869 4870 4871 4872 4873 4874
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
  return pOperator;

_error:
4875
  destroyStreamStateOperatorInfo(pInfo);
5
54liuyao 已提交
4876 4877 4878 4879
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
4880

4881
void destroyMergeAlignedIntervalOperatorInfo(void* param) {
4882
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param;
4883
  destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo);
D
dapan1121 已提交
4884
  taosMemoryFreeClear(param);
4885 4886
}

4887 4888
static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId,
                                                SSDataBlock* pResultBlock, TSKEY wstartTs) {
4889
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
4890

4891 4892 4893
  SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*            pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*                pSup = &pOperatorInfo->exprSupp;
4894

4895
  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId);
L
Liu Jicong 已提交
4896 4897
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(
      iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
4898
  ASSERT(p1 != NULL);
4899

4900 4901
  finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pSup->pCtx, pSup->pExprInfo, pSup->numOfExprs,
                                       pSup->rowEntryInfoOffset, pResultBlock, pTaskInfo);
4902 4903
  tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
  ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0);
4904

4905
  return TSDB_CODE_SUCCESS;
4906 4907
}

4908 4909
static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
                                          SSDataBlock* pBlock, int32_t scanFlag, SSDataBlock* pResultBlock) {
4910
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
D
dapan1121 已提交
4911
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
4912 4913

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
4914
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
4915 4916

  int32_t     startPos = 0;
4917
  int32_t     numOfOutput = pSup->numOfExprs;
4918
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
4919 4920 4921
  uint64_t    tableGroupId = pBlock->info.groupId;
  SResultRow* pResult = NULL;

4922 4923
  TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols);

4924 4925
  // there is an result exists
  if (miaInfo->curTs != INT64_MIN) {
4926
    ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1);
4927 4928

    if (ts != miaInfo->curTs) {
4929
      outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs);
4930
      miaInfo->curTs = ts;
4931
    }
4932 4933
  } else {
    miaInfo->curTs = ts;
4934
    ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0);
4935 4936 4937
  }

  STimeWindow win = {0};
4938
  win.skey = miaInfo->curTs;
4939 4940
  win.ekey =
      taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1;
4941

4942
  // TODO: remove the hash table (groupid + winkey => result row position)
4943 4944
  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
4945
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
4946
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
4947 4948
  }

4949 4950
  int32_t currPos = startPos;

4951
  STimeWindow currWin = win;
4952
  while (++currPos < pBlock->info.rows) {
4953
    if (tsCols[currPos] == miaInfo->curTs) {
4954
      continue;
4955 4956 4957
    }

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
4958 4959
    doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
                     pBlock->info.rows, numOfOutput);
4960

4961 4962
    outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs);
    miaInfo->curTs = tsCols[currPos];
4963

4964
    currWin.skey = miaInfo->curTs;
4965
    currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit,
4966 4967
                               iaInfo->interval.precision) -
                   1;
4968 4969 4970 4971 4972

    startPos = currPos;
    ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
4973
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
4974
    }
4975 4976

    miaInfo->curTs = currWin.skey;
4977
  }
4978

4979
  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
4980 4981
  doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
                   pBlock->info.rows, numOfOutput);
4982 4983
}

4984
static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
S
shenglian zhou 已提交
4985
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
4986

4987
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperator->info;
D
dapan1121 已提交
4988
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
4989

4990
  SExprSupp*   pSup = &pOperator->exprSupp;
4991
  SSDataBlock* pRes = iaInfo->binfo.pRes;
4992

4993 4994
  SOperatorInfo* downstream = pOperator->pDownstream[0];
  int32_t        scanFlag = MAIN_SCAN;
4995

4996 4997 4998 4999 5000 5001 5002
  while (1) {
    SSDataBlock* pBlock = NULL;
    if (miaInfo->prefetchedBlock == NULL) {
      pBlock = downstream->fpSet.getNextFn(downstream);
    } else {
      pBlock = miaInfo->prefetchedBlock;
      miaInfo->prefetchedBlock = NULL;
5003

5004 5005
      miaInfo->groupId = pBlock->info.groupId;
    }
5006

5007 5008 5009
    if (pBlock == NULL) {
      // close last unfinalized time window
      if (miaInfo->curTs != INT64_MIN) {
5010
        ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1);
5011 5012
        outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs);
        miaInfo->curTs = INT64_MIN;
5013
      }
5014

5015 5016
      doSetOperatorCompleted(pOperator);
      break;
5017
    }
5018

5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029
    if (!miaInfo->hasGroupId) {
      miaInfo->hasGroupId = true;
      miaInfo->groupId = pBlock->info.groupId;
    } else if (miaInfo->groupId != pBlock->info.groupId) {
      // if there are unclosed time window, close it firstly.
      ASSERT(miaInfo->curTs != INT64_MIN);
      outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs);
      miaInfo->prefetchedBlock = pBlock;
      miaInfo->curTs = INT64_MIN;
      break;
    }
5030

5031 5032 5033 5034 5035 5036 5037 5038
    getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag);
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true);
    doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);

    doFilter(miaInfo->pCondition, pRes, NULL);
    if (pRes->info.rows >= pOperator->resultInfo.capacity) {
      break;
    }
5039 5040
  }

5041
  pRes->info.groupId = miaInfo->groupId;
5042
  miaInfo->hasGroupId = false;
5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057
}

static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info;
  SIntervalAggOperatorInfo*             iaInfo = pMiaInfo->intervalAggOperatorInfo;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);

  if (iaInfo->binfo.mergeResultBlock) {
dengyihao's avatar
dengyihao 已提交
5058
    while (1) {
5059
      if (pOperator->status == OP_EXEC_DONE) {
5060 5061
        break;
      }
5062

5063
      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
5064 5065 5066
        break;
      }

5067 5068 5069 5070
      doMergeAlignedIntervalAgg(pOperator);
    }
  } else {
    doMergeAlignedIntervalAgg(pOperator);
5071 5072 5073 5074 5075 5076 5077
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

5078
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode,
5079
                                                      SExecTaskInfo* pTaskInfo) {
5080
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo));
5081
  SOperatorInfo*                        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5082 5083 5084 5085
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

D
dapan1121 已提交
5086 5087 5088 5089 5090
  miaInfo->intervalAggOperatorInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
  if (miaInfo->intervalAggOperatorInfo == NULL) {
    goto _error;
  }

5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101
  int32_t      num = 0;
  SExprInfo*   pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pNode->window.node.pOutputDataBlockDesc);

  SInterval interval = {.interval = pNode->interval,
                        .sliding = pNode->sliding,
                        .intervalUnit = pNode->intervalUnit,
                        .slidingUnit = pNode->slidingUnit,
                        .offset = pNode->offset,
                        .precision = ((SColumnNode*)pNode->window.pTspk)->node.resType.precision};

D
dapan1121 已提交
5102
  SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
5103
  SExprSupp*                pSup = &pOperator->exprSupp;
5104

5105
  miaInfo->pCondition = pNode->window.node.pConditions;
L
Liu Jicong 已提交
5106 5107 5108 5109 5110
  miaInfo->curTs = INT64_MIN;
  iaInfo->win = pTaskInfo->window;
  iaInfo->inputOrder = TSDB_ORDER_ASC;
  iaInfo->interval = interval;
  iaInfo->execModel = pTaskInfo->execModel;
5111 5112
  iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId;
  iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock;
5113 5114

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
5115
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5116

5117
  int32_t code = initAggInfo(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
5118 5119 5120
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
5121

5122
  initBasicInfo(&iaInfo->binfo, pResBlock);
5123
  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
5124

5125
  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, iaInfo);
5126
  if (iaInfo->timeWindowInterpo) {
5127
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
5128 5129
  }

5130
  initResultRowInfo(&iaInfo->binfo.resultRowInfo);
5131
  blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity);
5132

L
Liu Jicong 已提交
5133
  pOperator->name = "TimeMergeAlignedIntervalAggOperator";
5134
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL;
L
Liu Jicong 已提交
5135 5136 5137 5138
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = miaInfo;
5139

5140
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL,
5141
                                         destroyMergeAlignedIntervalOperatorInfo, NULL, NULL, NULL);
5142 5143 5144 5145 5146 5147 5148 5149 5150

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5151
  destroyMergeAlignedIntervalOperatorInfo(miaInfo);
5152 5153 5154 5155
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5156 5157 5158 5159 5160

//=====================================================================================================================
// merge interval operator
typedef struct SMergeIntervalAggOperatorInfo {
  SIntervalAggOperatorInfo intervalAggOperatorInfo;
L
Liu Jicong 已提交
5161 5162 5163 5164 5165 5166
  SList*                   groupIntervals;
  SListIter                groupIntervalsIter;
  bool                     hasGroupId;
  uint64_t                 groupId;
  SSDataBlock*             prefetchedBlock;
  bool                     inputBlocksFinished;
5167 5168
} SMergeIntervalAggOperatorInfo;

S
slzhou 已提交
5169
typedef struct SGroupTimeWindow {
L
Liu Jicong 已提交
5170
  uint64_t    groupId;
S
slzhou 已提交
5171 5172 5173
  STimeWindow window;
} SGroupTimeWindow;

5174
void destroyMergeIntervalOperatorInfo(void* param) {
5175
  SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
S
slzhou 已提交
5176
  tdListFree(miaInfo->groupIntervals);
5177
  destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo);
5178

D
dapan1121 已提交
5179
  taosMemoryFreeClear(param);
5180 5181
}

L
Liu Jicong 已提交
5182 5183
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win,
                                    SSDataBlock* pResultBlock) {
5184 5185 5186
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
5187
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5188 5189 5190
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId);
L
Liu Jicong 已提交
5191 5192
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(
      iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
5193 5194 5195
  ASSERT(p1 != NULL);
  finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo,
                                       pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo);
5196
  tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
5197 5198 5199
  return TSDB_CODE_SUCCESS;
}

5200 5201 5202 5203 5204
static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock,
                                        STimeWindow* newWin) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
5205
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5206 5207
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

S
slzhou 已提交
5208 5209
  SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin};
  tdListAppend(miaInfo->groupIntervals, &groupTimeWindow);
5210

S
slzhou 已提交
5211 5212 5213 5214 5215
  SListIter iter = {0};
  tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD);
  SListNode* listNode = NULL;
  while ((listNode = tdListNext(&iter)) != NULL) {
    SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
L
Liu Jicong 已提交
5216
    if (prevGrpWin->groupId != tableGroupId) {
S
slzhou 已提交
5217 5218 5219
      continue;
    }
    STimeWindow* prevWin = &prevGrpWin->window;
H
Haojun Liao 已提交
5220
    if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) {
S
slzhou 已提交
5221 5222 5223
      finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
      tdListPopNode(miaInfo->groupIntervals, listNode);
    }
5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240
  }

  return 0;
}

static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                                   int32_t scanFlag, SSDataBlock* pResultBlock) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*     pExprSup = &pOperatorInfo->exprSupp;

  int32_t     startPos = 0;
  int32_t     numOfOutput = pExprSup->numOfExprs;
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
5241
  bool        ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5242 5243 5244
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

5245 5246
  STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval,
                                        iaInfo->inputOrder);
5247 5248 5249 5250 5251

  int32_t ret =
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                             numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
5252
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5253 5254 5255 5256
  }

  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
5257
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5258 5259 5260 5261
  ASSERT(forwardRows > 0);

  // prev time window not interpolation yet.
  if (iaInfo->timeWindowInterpo) {
5262
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
5263 5264 5265 5266 5267 5268
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);

    // restore current time window
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                                 numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS) {
5269
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5270 5271 5272 5273 5274 5275 5276
    }

    // window start key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup);
  }

  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true);
5277 5278
  doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
                   pBlock->info.rows, numOfOutput);
5279 5280 5281 5282 5283 5284 5285 5286
  doCloseWindow(pResultRowInfo, iaInfo, pResult);

  // output previous interval results after this interval (&win) is closed
  outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win);

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = forwardRows - 1 + startPos;
5287 5288
    startPos =
        getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->inputOrder);
5289 5290 5291 5292 5293 5294 5295 5296 5297
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
    int32_t code =
        setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                               pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
5298
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5299 5300 5301 5302
    }

    ekey = ascScan ? nextWin.ekey : nextWin.skey;
    forwardRows =
5303
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5304 5305 5306 5307 5308

    // window start(end) key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup);

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true);
5309 5310
    doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pBlock->info.rows, numOfOutput);
5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350
    doCloseWindow(pResultRowInfo, iaInfo, pResult);

    // output previous interval results after this interval (&nextWin) is closed
    outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin);
  }

  if (iaInfo->timeWindowInterpo) {
    saveDataBlockLastRow(iaInfo->pPrevValues, pBlock, iaInfo->pInterpCols);
  }
}

static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExprSupp*                     pExpSupp = &pOperator->exprSupp;

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);

  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
        miaInfo->groupId = pBlock->info.groupId;
        miaInfo->prefetchedBlock = NULL;
      }

      if (pBlock == NULL) {
S
slzhou 已提交
5351
        tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD);
5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363
        miaInfo->inputBlocksFinished = true;
        break;
      }

      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
        miaInfo->groupId = pBlock->info.groupId;
      } else if (miaInfo->groupId != pBlock->info.groupId) {
        miaInfo->prefetchedBlock = pBlock;
        break;
      }

5364 5365
      getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag);
      setInputDataBlock(pOperator, pExpSupp->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true);
5366 5367 5368 5369 5370 5371 5372 5373
      doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);

      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
        break;
      }
    }

    pRes->info.groupId = miaInfo->groupId;
5374 5375 5376
  }

  if (miaInfo->inputBlocksFinished) {
S
slzhou 已提交
5377
    SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter);
5378

S
slzhou 已提交
5379 5380 5381 5382
    if (listNode != NULL) {
      SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data);
      finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
      pRes->info.groupId = grpWin->groupId;
5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394
    }
  }

  if (pRes->info.rows == 0) {
    doSetOperatorCompleted(pOperator);
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

5395 5396 5397
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode,
                                               SExecTaskInfo* pTaskInfo) {
  SMergeIntervalAggOperatorInfo* pMergeIntervalInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
5398
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5399
  if (pMergeIntervalInfo == NULL || pOperator == NULL) {
5400 5401 5402
    goto _error;
  }

5403 5404 5405 5406 5407 5408 5409 5410 5411 5412
  int32_t      num = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pIntervalPhyNode->window.node.pOutputDataBlockDesc);

  SInterval interval = {.interval = pIntervalPhyNode->interval,
                        .sliding = pIntervalPhyNode->sliding,
                        .intervalUnit = pIntervalPhyNode->intervalUnit,
                        .slidingUnit = pIntervalPhyNode->slidingUnit,
                        .offset = pIntervalPhyNode->offset,
                        .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
5413

5414
  pMergeIntervalInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow));
5415

5416
  SIntervalAggOperatorInfo* pIntervalInfo = &pMergeIntervalInfo->intervalAggOperatorInfo;
L
Liu Jicong 已提交
5417
  pIntervalInfo->win = pTaskInfo->window;
5418
  pIntervalInfo->inputOrder = TSDB_ORDER_ASC;
L
Liu Jicong 已提交
5419 5420
  pIntervalInfo->interval = interval;
  pIntervalInfo->execModel = pTaskInfo->execModel;
5421 5422
  pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock;
  pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
5423 5424 5425 5426

  SExprSupp* pExprSupp = &pOperator->exprSupp;

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
5427
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5428

5429
  int32_t code = initAggInfo(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
5430 5431 5432
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
5433

5434 5435
  initBasicInfo(&pIntervalInfo->binfo, pResBlock);
  initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win);
5436

5437 5438
  pIntervalInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, num, pIntervalInfo);
  if (pIntervalInfo->timeWindowInterpo) {
5439
    pIntervalInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
5440
    if (pIntervalInfo->binfo.resultRowInfo.openWindow == NULL) {
5441 5442 5443 5444
      goto _error;
    }
  }

5445
  initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo);
5446

L
Liu Jicong 已提交
5447
  pOperator->name = "TimeMergeIntervalAggOperator";
5448
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL;
L
Liu Jicong 已提交
5449 5450 5451 5452
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pMergeIntervalInfo;
5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL,
                                         destroyMergeIntervalOperatorInfo, NULL, NULL, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5465
  destroyMergeIntervalOperatorInfo(pMergeIntervalInfo);
5466 5467 5468
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
5469
}
5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777

static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
    SSDataBlock* pBlock, int32_t scanFlag, SHashObj* pUpdatedMap) {
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;

  int32_t     startPos = 0;
  int32_t     numOfOutput = pSup->numOfExprs;
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
  TSKEY*      tsCols = (TSKEY*)pColDataInfo->pData;
  uint64_t    tableGroupId = pBlock->info.groupId;
  bool        ascScan = true;
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

  STimeWindow win =
      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
  int32_t ret = TSDB_CODE_SUCCESS;
  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
    }
  }

  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
  ASSERT(forwardRows > 0);

  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
  }

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = forwardRows - 1 + startPos;
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);
    if (startPos < 0) {
      break;
    }
    if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
      ekey = ascScan ? nextWin.ekey : nextWin.skey;
      forwardRows =
          getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
      continue;
    }

    // null data, failed to allocate more memory buffer
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
    }

    ekey = ascScan ? nextWin.ekey : nextWin.skey;
    forwardRows =
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
  }
}

static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
  int64_t                   maxTs = INT64_MIN;
  SExprSupp* pSup = &pOperator->exprSupp;

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows > 0) {
      printDataBlock(pInfo->pDelRes, "single interval");
      return pInfo->pDelRes;
    }

    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
    if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
      pOperator->status = OP_EXEC_DONE;
      qDebug("===stream===single interval is done");
      freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
    }
    printDataBlock(pInfo->binfo.pRes, "single interval");
    return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];

  SArray*    pUpdated = taosArrayInit(4, POINTER_BYTES);  // SResKeyPos
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*  pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);

  SStreamState* pState = pTaskInfo->streamInfo.pState;

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
    printDataBlock(pBlock, "single interval recv");

    if (pBlock->info.type == STREAM_CLEAR) {
      doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock,
                     NULL);
      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
      continue;
    } else if (pBlock->info.type == STREAM_DELETE_DATA) {
      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval, pUpdatedMap);
      continue;
    } else if (pBlock->info.type == STREAM_GET_ALL) {
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
      continue;
    }

    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {
      // set input version
      pTaskInfo->version = pBlock->info.version;
    }

    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }

    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
    // caller. Note that all the time window are not close till now.
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
    if (pInfo->invertible) {
      setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type);
    }

    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
    doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap);
  }
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);

#if 0
  if (pState) {
    printf(">>>>>>>> stream read backend\n");
    SWinKey key = {
        .ts = 1,
        .groupId = 2,
    };
    char*   val = NULL;
    int32_t sz;
    if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) {
      ASSERT(0);
    }
    printf("stream read %s %d\n", val, sz);
    streamFreeVal(val);

    SStreamStateCur* pCur = streamStateGetCur(pState, &key);
    ASSERT(pCur);
    while (streamStateCurNext(pState, pCur) == 0) {
      SWinKey     key1;
      const void* val1;
      if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) {
        break;
      }
      printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz);
    }
    streamStateFreeCur(pCur);
  }
#endif

  pOperator->status = OP_RES_TO_RETURN;
  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap,
                      pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);

  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosArraySort(pUpdated, resultrowComparAsc);

  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
  taosHashCleanup(pUpdatedMap);
  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows > 0) {
    printDataBlock(pInfo->pDelRes, "single interval");
    return pInfo->pDelRes;
  }

  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
  printDataBlock(pInfo->binfo.pRes, "single interval");
  return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}

void destroyStreamIntervalOperatorInfo(void* param) {
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;
  cleanupBasicInfo(&pInfo->binfo);
  cleanupAggSup(&pInfo->aggSup);
  pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages);

  pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins);
  pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);

  cleanupGroupResInfo(&pInfo->groupResInfo);
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  taosMemoryFreeClear(param);
}

SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

  int32_t numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
  ASSERT(numOfCols > 0);
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  SInterval interval = {.interval = pIntervalPhyNode->interval,
                        .sliding = pIntervalPhyNode->sliding,
                        .intervalUnit = pIntervalPhyNode->intervalUnit,
                        .slidingUnit = pIntervalPhyNode->slidingUnit,
                        .offset = pIntervalPhyNode->offset,
                        .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, };
  STimeWindowAggSupp twAggSupp = {.waterMark = pIntervalPhyNode->window.watermark,
                                  .calTrigger = pIntervalPhyNode->window.triggerType,
                                  .maxTs = INT64_MIN, };
  ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY);
  pOperator->pTaskInfo = pTaskInfo;
  pInfo->interval = interval;
  pInfo->twAggSup = twAggSupp;
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
  pInfo->isFinal = false;

  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;;
  initResultSizeInfo(&pOperator->resultInfo, 4096);
  SExprSupp* pSup = &pOperator->exprSupp;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  initBasicInfo(&pInfo->binfo, pResBlock);
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
  pInfo->delIndex = 0;
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
  initResultRowInfo(&pInfo->binfo.resultRowInfo);

  pOperator->name = "StreamIntervalOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL,
                                         destroyStreamIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
  destroyStreamIntervalOperatorInfo(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}