timewindowoperator.c 223.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15
#include "executorimpl.h"
X
Xiaoyu Wang 已提交
16
#include "function.h"
5
54liuyao 已提交
17
#include "functionMgt.h"
H
Haojun Liao 已提交
18
#include "tcommon.h"
5
54liuyao 已提交
19
#include "tcompare.h"
L
Liu Jicong 已提交
20
#include "tdatablock.h"
H
Haojun Liao 已提交
21
#include "tfill.h"
22
#include "ttime.h"
23 24 25 26 27 28

typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;

5
54liuyao 已提交
29 30
#define IS_FINAL_OP(op) ((op)->isFinal)

5
54liuyao 已提交
31 32
typedef struct SPullWindowInfo {
  STimeWindow window;
L
Liu Jicong 已提交
33
  uint64_t    groupId;
5
54liuyao 已提交
34 35
} SPullWindowInfo;

36 37
typedef struct SOpenWindowInfo {
  SResultRowPosition pos;
L
Liu Jicong 已提交
38
  uint64_t           groupId;
39 40
} SOpenWindowInfo;

5
54liuyao 已提交
41
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
5
54liuyao 已提交
42

43 44
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);

L
Liu Jicong 已提交
45 46
static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult,
                                              uint64_t groupId);
47 48
static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult);

X
Xiaoyu Wang 已提交
49
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }
50 51 52

static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
53
                                      int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
54 55 56 57 58 59 60 61 62 63 64
                                      SExecTaskInfo* pTaskInfo) {
  SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                  masterscan, tableGroupId, pTaskInfo, true, pAggSup);

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // set time window for current result
  pResultRow->win = (*win);
65

66
  *pResult = pResultRow;
67
  setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
68

69 70 71 72 73 74 75 76 77 78 79 80 81
  return TSDB_CODE_SUCCESS;
}

static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) {
  int64_t* ts = (int64_t*)pColData->pData;
  int32_t  delta = includeEndpoint ? 1 : 0;

  int64_t duration = pWin->ekey - pWin->skey + delta;
  ts[2] = duration;            // set the duration
  ts[3] = pWin->skey;          // window start key
  ts[4] = pWin->ekey + delta;  // window end key
}

82
static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) {
83 84 85
  pRowSup->win.ekey = ts;
  pRowSup->prevTs = ts;
  pRowSup->numOfRows += 1;
86
  pRowSup->groupId = groupId;
87 88
}

dengyihao's avatar
dengyihao 已提交
89 90
static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex,
                                     uint64_t groupId) {
91 92 93
  pRowSup->startRowIndex = rowIndex;
  pRowSup->numOfRows = 0;
  pRowSup->win.skey = tsList[rowIndex];
94
  pRowSup->groupId = groupId;
95 96 97 98
}

static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
                                                   int16_t pos, int16_t order, int64_t* pData) {
99
  int32_t forwardRows = 0;
100 101 102 103

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
104
      forwardRows = end;
105 106

      if (pData[end + pos] == ekey) {
107
        forwardRows += 1;
108 109 110
      }
    }
  } else {
111
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
112
    if (end >= 0) {
113
      forwardRows = end;
114

115
      if (pData[end + pos] == ekey) {
116
        forwardRows += 1;
117 118
      }
    }
X
Xiaoyu Wang 已提交
119 120 121 122 123 124 125 126
    //    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
    //    if (end >= 0) {
    //      forwardRows = pos - end;
    //
    //      if (pData[end] == ekey) {
    //        forwardRows += 1;
    //      }
    //    }
127 128
  }

129 130
  assert(forwardRows >= 0);
  return forwardRows;
131 132
}

5
54liuyao 已提交
133
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY*  keyList = (TSKEY*)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
150 151 152 153 154 155 156 157 158 159 160
      if (key >= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key < keyList[lastPos]) {
        lastPos += 1;
        if (lastPos >= num) {
          return -1;
        } else {
          return lastPos;
        }
      }
161 162 163 164 165 166

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        firstPos = midPos + 1;
167 168
      } else if (key > keyList[midPos]) {
        lastPos = midPos - 1;
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

X
Xiaoyu Wang 已提交
204 205
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
                                 __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num = -1;
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);

  if (order == TSDB_ORDER_ASC) {
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
      if (item != NULL) {
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
227
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
228 229
      }
    } else {
230
      num = pDataBlockInfo->rows - startPos;
231
      if (item != NULL) {
232
        item->lastKey = pDataBlockInfo->window.ekey + step;
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
      }
    }
  }

  assert(num >= 0);
  return num;
}

static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
    return;
  }

  int64_t key = tw->skey, interval = pInterval->interval;
  // convert key to second
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;

  if (pInterval->intervalUnit == 'y') {
    interval *= 12;
  }

  struct tm tm;
  time_t    t = (time_t)key;
  taosLocalTime(&t, &tm);

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
264
  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
265 266 267 268

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
269
  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
270 271 272 273

  tw->ekey -= 1;
}

5
54liuyao 已提交
274 275 276 277
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) {
  getNextTimeWindow(pInterval, pInterval->precision, order, tw);
}

278 279
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs,
                               int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
280
  SqlFunctionCtx* pCtx = pSup->pCtx;
281

282
  int32_t index = 1;
283
  for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
H
Haojun Liao 已提交
284
    if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
285 286 287 288
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

X
Xiaoyu Wang 已提交
289
    SFunctParam*     pParam = &pCtx[k].param[0];
290 291
    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId);

292
    ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey);
293

294
    double v1 = 0, v2 = 0, v = 0;
295
    if (prevRowIndex == -1) {
296
      SGroupKeys* p = taosArrayGet(pPrevValues, index);
297
      GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData);
298
    } else {
299
      GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex));
300 301
    }

302
    GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
303

304
#if 0
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            //            pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
324 325
#endif

X
Xiaoyu Wang 已提交
326 327 328
    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
    SPoint point = (SPoint){.key = windowKey, .val = &v};
329

X
Xiaoyu Wang 已提交
330
    taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
331

X
Xiaoyu Wang 已提交
332 333 334 335 336 337
    if (type == RESULT_ROW_START_INTERP) {
      pCtx[k].start.key = point.key;
      pCtx[k].start.val = v;
    } else {
      pCtx[k].end.key = point.key;
      pCtx[k].end.val = v;
338
    }
X
Xiaoyu Wang 已提交
339 340 341

    index += 1;
  }
342
#if 0
343
  }
344
#endif
345 346 347 348 349 350 351 352 353 354 355 356 357 358
}

static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

359 360
static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock,
                                              const TSKEY* tsCols, STimeWindow* win, SExprSupp* pSup) {
361
  bool ascQuery = (pInfo->inputOrder == TSDB_ORDER_ASC);
362

363
  TSKEY curTs = tsCols[pos];
364 365

  SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
X
Xiaoyu Wang 已提交
366
  TSKEY       lastTs = *(int64_t*)pTsKey->pData;
367 368 369 370 371

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery ? win->skey : win->ekey;
  if (key == curTs) {
372
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
373 374 375
    return true;
  }

376 377
  // it is the first time window, no need to do interpolation
  if (pTsKey->isNull && pos == 0) {
378
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
379 380
  } else {
    TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]);
381 382
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key,
                              RESULT_ROW_START_INTERP, pSup);
383 384 385 386 387
  }

  return true;
}

388 389 390
static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex,
                                            SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey,
                                            STimeWindow* win) {
391
  int32_t order = pInfo->inputOrder;
392 393

  TSKEY actualEndKey = tsCols[endRowIndex];
394
  TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey;
395 396

  // not ended in current data block, do not invoke interpolation
397
  if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) {
398
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
399 400 401
    return false;
  }

402
  // there is actual end point of current time window, no interpolation needs
403
  if (key == actualEndKey) {
404
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
405 406 407
    return true;
  }

408
  int32_t nextRowIndex = endRowIndex + 1;
409 410 411
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
412 413
  doTimeWindowInterpolation(pInfo->pPrevValues, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key,
                            RESULT_ROW_END_INTERP, pSup);
414 415 416
  return true;
}

5
54liuyao 已提交
417
bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo) {
418 419
  if (pInterval->interval != pInterval->sliding &&
      (pWin->ekey < pBlockInfo->calWin.skey || pWin->skey > pBlockInfo->calWin.ekey)) {
5
54liuyao 已提交
420 421 422 423 424
    return false;
  }
  return true;
}

425
static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
5
54liuyao 已提交
426
                                      TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
X
Xiaoyu Wang 已提交
427
  bool ascQuery = (order == TSDB_ORDER_ASC);
428 429 430 431 432 433 434 435 436 437

  int32_t precision = pInterval->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
    return -1;
  }

5
54liuyao 已提交
438 439 440 441
  if (!inSlidingWindow(pInterval, pNext, pDataBlockInfo) && order == TSDB_ORDER_ASC) {
    return -1;
  }

442
  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
443 444 445 446
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
447
    startPos = prevPosition + 1;
448
  } else {
449
    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
450 451
      startPos = 0;
    } else {
452
      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
    }
  }

  /* interp query with fill should not skip time window */
  //  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
  //    return startPos;
  //  }

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
    if (ascQuery) {
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
      }
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
      }
    }
  }

  return startPos;
}

496 497
static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
  ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp == true;
  }
}

static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp = true;
  }
}

514 515
static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult,
                                        STimeWindow* win, int32_t startPos, int32_t forwardRows, SExprSupp* pSup) {
516
  if (!pInfo->timeWindowInterpo) {
517 518 519
    return;
  }

520
  ASSERT(pBlock != NULL);
521 522 523 524 525
  if (pBlock->pDataBlock == NULL) {
    //    tscError("pBlock->pDataBlock == NULL");
    return;
  }

526
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
527 528

  TSKEY* tsCols = (TSKEY*)(pColInfo->pData);
529
  bool   done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
530
  if (!done) {  // it is not interpolated, now start to generated the interpolated value
531
    bool interp = setTimeWindowInterpolationStartTs(pInfo, startPos, pBlock, tsCols, win, pSup);
532 533 534 535
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
536
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
537 538 539 540 541 542 543 544
  }

  // point interpolation does not require the end key time window interpolation.
  //  if (pointInterpQuery) {
  //    return;
  //  }

  // interpolation query does not generate the time window end interpolation
545
  done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
546
  if (!done) {
547
    int32_t endRowIndex = startPos + forwardRows - 1;
548

549
    TSKEY endKey = (pInfo->inputOrder == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
550
    bool  interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
551 552 553 554
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
555
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
556 557 558
  }
}

559 560
static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) {
  if (pBlock->pDataBlock == NULL) {
561 562 563
    return;
  }

564 565 566 567 568 569 570
  size_t num = taosArrayGetSize(pPrevKeys);
  for (int32_t k = 0; k < num; ++k) {
    SColumn* pc = taosArrayGet(pCols, k);

    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId);

    SGroupKeys* pkey = taosArrayGet(pPrevKeys, k);
X
Xiaoyu Wang 已提交
571
    for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) {
572 573 574 575 576 577 578 579 580 581 582 583 584 585
      if (colDataIsNull_s(pColInfo, i)) {
        continue;
      }

      char* val = colDataGetData(pColInfo, i);
      if (IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, varDataTLen(val));
        ASSERT(varDataTLen(val) <= pkey->bytes);
      } else {
        memcpy(pkey->pData, val, pkey->bytes);
      }

      break;
    }
586 587 588
  }
}

589 590 591 592
static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo,
                                       SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) {
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

593
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
594
  SExprSupp*                pSup = &pOperatorInfo->exprSupp;
595

L
Liu Jicong 已提交
596 597
  int32_t startPos = 0;
  int32_t numOfOutput = pSup->numOfExprs;
598

599
  SResultRow* pResult = NULL;
600

601
  while (1) {
L
Liu Jicong 已提交
602 603 604
    SListNode*          pn = tdListGetHead(pResultRowInfo->openWindow);
    SOpenWindowInfo*    pOpenWin = (SOpenWindowInfo*)pn->data;
    uint64_t            groupId = pOpenWin->groupId;
605
    SResultRowPosition* p1 = &pOpenWin->pos;
606 607 608
    if (p->pageId == p1->pageId && p->offset == p1->offset) {
      break;
    }
609

610
    SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1, false);
611
    ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
612

613
    if (pr->closed) {
X
Xiaoyu Wang 已提交
614 615
      ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
             isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
616 617
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
618 619
      continue;
    }
620

621
    STimeWindow w = pr->win;
622 623
    int32_t     ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx,
                                             numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
624
    if (ret != TSDB_CODE_SUCCESS) {
625
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
626 627 628 629
    }

    ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP));

X
Xiaoyu Wang 已提交
630 631
    SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
    int64_t     prevTs = *(int64_t*)pTsKey->pData;
632 633 634 635
    if (groupId == pBlock->info.groupId) {
      doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey,
                                RESULT_ROW_END_INTERP, pSup);
    }
636 637

    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
638
    setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
639

640
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, true);
H
Haojun Liao 已提交
641 642
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, pBlock->info.rows,
                     numOfExprs);
643 644 645

    if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
      closeResultRow(pr);
646 647
      SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
      taosMemoryFree(pNode);
X
Xiaoyu Wang 已提交
648
    } else {  // the remains are can not be closed yet.
649
      break;
650
    }
651
  }
652
}
653

5
54liuyao 已提交
654
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
5
54liuyao 已提交
655
  if (!pBlock || pBlock->info.rows == 0) {
5
54liuyao 已提交
656
    qDebug("===stream===printDataBlock: Block is Null or Empty");
5
54liuyao 已提交
657 658
    return;
  }
L
Liu Jicong 已提交
659
  char* pBuf = NULL;
5
54liuyao 已提交
660 661
  qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
  taosMemoryFree(pBuf);
5
54liuyao 已提交
662 663
}

5
54liuyao 已提交
664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719
typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);

int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;
      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;
      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;
      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

      if (comparefn(pKey, keyList, lastPos) > 0) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

5
54liuyao 已提交
720
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
721

X
Xiaoyu Wang 已提交
722 723 724
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;
5
54liuyao 已提交
725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (key >= getValuefn(keyList, lastPos)) return lastPos;
      if (key == getValuefn(keyList, firstPos)) return firstPos;
      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (key <= getValuefn(keyList, firstPos)) return firstPos;
      if (key == getValuefn(keyList, lastPos)) return lastPos;

      if (key > getValuefn(keyList, lastPos)) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
771 772
  }

5
54liuyao 已提交
773 774 775
  return midPos;
}

5
54liuyao 已提交
776
int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {
L
Liu Jicong 已提交
777
  SArray*          res = (SArray*)data;
5
54liuyao 已提交
778
  SPullWindowInfo* pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
779
  SPullWindowInfo* pData = (SPullWindowInfo*)pKey;
5
54liuyao 已提交
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810
  if (pData->window.skey == pos->window.skey) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->window.skey > pos->window.skey) {
    return 1;
  }
  return -1;
}

static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {
  int32_t size = taosArrayGetSize(pPullWins);
  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);
  if (index == -1) {
    index = 0;
  } else {
    if (comparePullWinKey(pPullInfo, pPullWins, index) > 0) {
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
811 812 813
int32_t compareResKey(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
  SResKeyPos* pos = taosArrayGetP(res, index);
H
Haojun Liao 已提交
814
  SWinKey*    pData = (SWinKey*)pKey;
5
54liuyao 已提交
815 816 817 818 819 820 821 822 823 824 825 826 827
  if (pData->ts == *(int64_t*)pos->key) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->ts > *(int64_t*)pos->key) {
    return 1;
  }
  return -1;
}

828
static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SArray* pUpdated) {
5
54liuyao 已提交
829
  int32_t size = taosArrayGetSize(pUpdated);
H
Haojun Liao 已提交
830
  SWinKey data = {.ts = ts, .groupId = groupId};
5
54liuyao 已提交
831
  int32_t index = binarySearchCom(pUpdated, size, &data, TSDB_ORDER_DESC, compareResKey);
5
54liuyao 已提交
832 833 834
  if (index == -1) {
    index = 0;
  } else {
5
54liuyao 已提交
835
    if (compareResKey(&data, pUpdated, index) > 0) {
5
54liuyao 已提交
836 837 838 839 840
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
H
Haojun Liao 已提交
841

5
54liuyao 已提交
842 843 844 845 846
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
5
54liuyao 已提交
847 848
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
X
Xiaoyu Wang 已提交
849
  if (taosArrayInsert(pUpdated, index, &newPos) == NULL) {
5
54liuyao 已提交
850 851 852 853 854
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
855 856 857 858 859 860 861 862
static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SHashObj* pUpdatedMap) {
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
H
Haojun Liao 已提交
863 864
  SWinKey key = {.ts = ts, .groupId = groupId};
  if (taosHashPut(pUpdatedMap, &key, sizeof(SWinKey), &newPos, sizeof(void*)) != TSDB_CODE_SUCCESS) {
5
54liuyao 已提交
865 866 867
    taosMemoryFree(newPos);
  }
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
868 869
}

5
54liuyao 已提交
870 871 872 873
static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SHashObj* pUpdatedMap) {
  return saveWinResult(ts, -1, -1, groupId, pUpdatedMap);
}

5
54liuyao 已提交
874 875
static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
  return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated);
5
54liuyao 已提交
876 877
}

5
54liuyao 已提交
878
static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) {
5
54liuyao 已提交
879 880
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
H
Haojun Liao 已提交
881 882
    SWinKey* pW = taosArrayGet(pWins, i);
    taosHashRemove(pUpdatedMap, pW, sizeof(SWinKey));
5
54liuyao 已提交
883 884 885
  }
}

886
int64_t getWinReskey(void* data, int32_t index) {
887
  SArray*  res = (SArray*)data;
H
Haojun Liao 已提交
888
  SWinKey* pos = taosArrayGet(res, index);
889 890 891
  return pos->ts;
}

5
54liuyao 已提交
892 893
int32_t compareWinRes(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
894
  SWinKey*    pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
895
  SResKeyPos* pData = (SResKeyPos*)pKey;
5
54liuyao 已提交
896 897 898 899 900 901 902 903 904 905 906 907 908
  if (*(int64_t*)pData->key == pos->ts) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (*(int64_t*)pData->key > pos->ts) {
    return 1;
  }
  return -1;
}

5
54liuyao 已提交
909
static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) {
L
Liu Jicong 已提交
910 911
  int32_t delSize = taosArrayGetSize(pDelWins);
  if (taosHashGetSize(pUpdatedMap) == 0 || delSize == 0) {
5
54liuyao 已提交
912
    return;
dengyihao's avatar
dengyihao 已提交
913
  }
L
Liu Jicong 已提交
914
  void* pIte = NULL;
5
54liuyao 已提交
915
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
916
    SResKeyPos* pResKey = *(SResKeyPos**)pIte;
5
54liuyao 已提交
917 918
    int32_t     index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes);
    if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) {
919
      taosArrayRemove(pDelWins, index);
920
      delSize = taosArrayGetSize(pDelWins);
921 922 923 924
    }
  }
}

5
54liuyao 已提交
925 926 927
bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) {
  ASSERT(pTwSup->maxTs == INT64_MIN || pTwSup->maxTs > 0);
  return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark;
5
54liuyao 已提交
928 929
}

5
54liuyao 已提交
930 931 932 933 934
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); }

bool needDeleteWindowBuf(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) {
  return pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark;
}
5
54liuyao 已提交
935

5
54liuyao 已提交
936
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
937
                            int32_t scanFlag) {
938
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
939

940
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
941
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
942

X
Xiaoyu Wang 已提交
943
  int32_t     startPos = 0;
944
  int32_t     numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
945 946
  int64_t*    tsCols = extractTsCol(pBlock, pInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
947
  bool        ascScan = (pInfo->inputOrder == TSDB_ORDER_ASC);
X
Xiaoyu Wang 已提交
948 949
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;
950

951 952
  STimeWindow win =
      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder);
953 954
  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
955 956
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
957
  }
X
Xiaoyu Wang 已提交
958 959
  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
960
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
961
  ASSERT(forwardRows > 0);
962 963

  // prev time window not interpolation yet.
964
  if (pInfo->timeWindowInterpo) {
965
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
966
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);
967 968

    // restore current time window
969 970
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
971
    if (ret != TSDB_CODE_SUCCESS) {
972
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
973 974
    }

975
    // window start key interpolation
976
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
977
  }
978

979 980
  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
981
                   numOfOutput);
982 983

  doCloseWindow(pResultRowInfo, pInfo, pResult);
984 985 986

  STimeWindow nextWin = win;
  while (1) {
987
    int32_t prevEndPos = forwardRows - 1 + startPos;
988
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->inputOrder);
989 990 991 992
    if (startPos < 0) {
      break;
    }
    // null data, failed to allocate more memory buffer
X
Xiaoyu Wang 已提交
993
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
994
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
995
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
996
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
997 998
    }

X
Xiaoyu Wang 已提交
999
    ekey = ascScan ? nextWin.ekey : nextWin.skey;
1000
    forwardRows =
1001
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
1002
    // window start(end) key interpolation
1003
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
L
Liu Jicong 已提交
1004
    // TODO: add to open window? how to close the open windows after input blocks exhausted?
S
shenglian zhou 已提交
1005
#if 0
1006 1007 1008 1009
    if ((ascScan && ekey <= pBlock->info.window.ekey) ||
        (!ascScan && ekey >= pBlock->info.window.skey)) {
      // window start(end) key interpolation
      doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
1010
    } else if (pInfo->timeWindowInterpo) {
1011 1012
      addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
    }
S
shenglian zhou 已提交
1013
#endif
1014
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
H
Haojun Liao 已提交
1015 1016
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
1017
    doCloseWindow(pResultRowInfo, pInfo, pResult);
1018 1019 1020
  }

  if (pInfo->timeWindowInterpo) {
1021
    saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
1022
  }
1023 1024 1025 1026 1027 1028 1029 1030 1031 1032
}

void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
  // current result is done in computing final results.
  if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
    closeResultRow(pResult);
    tdListPopHead(pResultRowInfo->openWindow);
  }
}

1033 1034 1035 1036 1037
SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, uint64_t groupId) {
  SOpenWindowInfo openWin = {0};
  openWin.pos.pageId = pResult->pageId;
  openWin.pos.offset = pResult->offset;
  openWin.groupId = groupId;
L
Liu Jicong 已提交
1038
  SListNode* pn = tdListGetTail(pResultRowInfo->openWindow);
1039
  if (pn == NULL) {
1040 1041
    tdListAppend(pResultRowInfo->openWindow, &openWin);
    return openWin.pos;
1042 1043
  }

L
Liu Jicong 已提交
1044
  SOpenWindowInfo* px = (SOpenWindowInfo*)pn->data;
1045 1046
  if (px->pos.pageId != openWin.pos.pageId || px->pos.offset != openWin.pos.offset || px->groupId != openWin.groupId) {
    tdListAppend(pResultRowInfo->openWindow, &openWin);
1047 1048
  }

1049
  return openWin.pos;
1050 1051 1052 1053
}

int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
  TSKEY* tsCols = NULL;
1054

1055 1056 1057 1058
  if (pBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;

1059 1060 1061 1062 1063 1064
    // no data in primary ts
    if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) {
      return NULL;
    }

    if (tsCols[0] != 0 && (pBlock->info.window.skey == 0 && pBlock->info.window.ekey == 0)) {
1065 1066 1067 1068 1069
      blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
    }
  }

  return tsCols;
1070 1071 1072 1073 1074 1075 1076
}

static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
  }

L
Liu Jicong 已提交
1077
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1078
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
1079
  SExprSupp*                pSup = &pOperator->exprSupp;
1080

1081 1082
  int32_t scanFlag = MAIN_SCAN;

X
Xiaoyu Wang 已提交
1083
  int64_t        st = taosGetTimestampUs();
1084 1085 1086
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1087
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1088 1089 1090 1091
    if (pBlock == NULL) {
      break;
    }

1092
    getTableScanInfo(pOperator, &pInfo->inputOrder, &scanFlag);
1093

1094
    if (pInfo->scalarSupp.pExprInfo != NULL) {
L
Liu Jicong 已提交
1095 1096
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
1097 1098
    }

1099
    // the pDataBlock are always the same one, no need to call this again
1100
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, scanFlag, true);
1101 1102
    blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);

1103
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag);
1104 1105
  }

1106
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->resultTsOrder);
1107
  OPTR_SET_OPENED(pOperator);
1108 1109

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1110 1111 1112
  return TSDB_CODE_SUCCESS;
}

1113 1114 1115 1116 1117
static bool compareVal(const char* v, const SStateKeys* pKey) {
  if (IS_VAR_DATA_TYPE(pKey->type)) {
    if (varDataLen(v) != varDataLen(pKey->pData)) {
      return false;
    } else {
D
dapan1121 已提交
1118
      return memcmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0;
1119 1120 1121 1122 1123 1124
    }
  } else {
    return memcmp(pKey->pData, v, pKey->bytes) == 0;
  }
}

1125
static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
L
Liu Jicong 已提交
1126
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1127
  SExprSupp*     pSup = &pOperator->exprSupp;
1128

1129
  SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
1130 1131 1132
  int64_t          gid = pBlock->info.groupId;

  bool    masterScan = true;
1133
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1134 1135
  int16_t bytes = pStateColInfoData->info.bytes;

1136
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1137 1138 1139 1140 1141
  TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

1142
  struct SColumnDataAgg* pAgg = NULL;
1143
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
X
Xiaoyu Wang 已提交
1144
    pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
1145
    if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
1146 1147 1148 1149 1150
      continue;
    }

    char* val = colDataGetData(pStateColInfoData, j);

1151
    if (gid != pRowSup->groupId || !pInfo->hasKey) {
1152 1153 1154 1155 1156 1157 1158
      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }

1159 1160
      pInfo->hasKey = true;

1161 1162
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1163
    } else if (compareVal(val, &pInfo->stateKey)) {
1164
      doKeepTuple(pRowSup, tsList[j], gid);
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // a new state window started
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1175 1176
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1177
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1178
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1179 1180 1181
      }

      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1182 1183
      doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                       pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1184 1185

      // here we start a new session window
1186 1187
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1188 1189 1190 1191 1192 1193 1194

      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }
1195 1196 1197 1198 1199
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1200 1201
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1202
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1203
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1204 1205 1206
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
H
Haojun Liao 已提交
1207 1208
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
                   pBlock->info.rows, numOfOutput);
1209 1210
}

1211
static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
1212 1213 1214 1215 1216
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
1217

1218 1219
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  SExprSupp*     pSup = &pOperator->exprSupp;
1220

1221
  SOptrBasicInfo* pBInfo = &pInfo->binfo;
1222 1223

  if (pOperator->status == OP_RES_TO_RETURN) {
1224
    while (1) {
1225
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1226
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1227

1228
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }

      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1239
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1240 1241
  }

1242
  int32_t order = TSDB_ORDER_ASC;
1243
  int64_t st = taosGetTimestampUs();
1244 1245 1246

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  while (1) {
1247
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1248 1249 1250 1251
    if (pBlock == NULL) {
      break;
    }

1252
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1253 1254
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1255 1256 1257
    doStateWindowAggImpl(pOperator, pInfo, pBlock);
  }

X
Xiaoyu Wang 已提交
1258
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1259 1260
  pOperator->status = OP_RES_TO_RETURN;

1261
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1262
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1263
  while (1) {
1264
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1265
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1266

1267
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1268 1269 1270 1271
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
1272

1273 1274 1275 1276 1277
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1278
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1279 1280
}

1281
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
1282
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
1283
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1284 1285 1286 1287 1288 1289 1290 1291

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pBlock = pInfo->binfo.pRes;

  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
1292
    return pOperator->fpSet.getStreamResFn(pOperator);
1293 1294 1295 1296 1297 1298 1299
  } else {
    pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
    if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
      return NULL;
    }

    blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
1300 1301
    while (1) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1302
      doFilter(pInfo->pCondition, pBlock, NULL);
1303

1304
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1305 1306 1307 1308
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
1309

1310 1311 1312
      if (pBlock->info.rows > 0) {
        break;
      }
1313 1314
    }

1315 1316 1317
    size_t rows = pBlock->info.rows;
    pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1318
    return (rows == 0) ? NULL : pBlock;
1319 1320 1321 1322
  }
}

// todo merged with the build group result.
1323
static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
1324
                                  int32_t* rowEntryInfoOffset) {
1325 1326 1327 1328 1329 1330 1331
  size_t num = taosArrayGetSize(pUpdateList);

  for (int32_t i = 0; i < num; ++i) {
    SResKeyPos* pPos = taosArrayGetP(pUpdateList, i);

    SFilePage*  bufPage = getBufPage(pBuf, pPos->pos.pageId);
    SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
1332

1333
    for (int32_t j = 0; j < numOfOutput; ++j) {
1334
      SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, j, rowEntryInfoOffset);
1335 1336
      if (pRow->numOfRows < pEntry->numOfRes) {
        pRow->numOfRows = pEntry->numOfRes;
1337 1338 1339 1340 1341 1342
      }
    }

    releaseBufPage(pBuf, bufPage);
  }
}
5
54liuyao 已提交
1343
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
L
Liu Jicong 已提交
1344
  for (int i = 0; i < num; i++) {
5
54liuyao 已提交
1345 1346
    if (type == STREAM_INVERT) {
      fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
L
Liu Jicong 已提交
1347
    } else if (type == STREAM_NORMAL) {
5
54liuyao 已提交
1348 1349 1350 1351
      fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
    }
  }
}
5
54liuyao 已提交
1352

5
54liuyao 已提交
1353
static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) {
1354
  SResultRow*     pResult = getResultRowByPos(pResultBuf, p1, false);
1355
  SqlFunctionCtx* pCtx = pSup->pCtx;
5
54liuyao 已提交
1356
  for (int32_t i = 0; i < numOfOutput; ++i) {
1357
    pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
1358 1359 1360 1361 1362 1363 1364 1365 1366
    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
      continue;
    }
    pResInfo->initialized = false;
    if (pCtx[i].functionId != -1) {
      pCtx[i].fpSet.init(&pCtx[i], pResInfo);
    }
  }
5
54liuyao 已提交
1367 1368 1369
  SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pResultBuf, bufPage);
5
54liuyao 已提交
1370 1371
}

5
54liuyao 已提交
1372 1373
static bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
                          int32_t numOfOutput) {
1374
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId);
5
54liuyao 已提交
1375
  SResultRowPosition* p1 =
1376
      (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1377 1378
  if (!p1) {
    // window has been closed
5
54liuyao 已提交
1379
    return false;
1380
  }
1381
  doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput);
5
54liuyao 已提交
1382
  return true;
5
54liuyao 已提交
1383 1384
}

5
54liuyao 已提交
1385 1386 1387 1388
static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, int32_t numOfOutput) {
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
  SWinKey                      key = {.ts = ts, .groupId = groupId};
  tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));
1389
  streamStateDel(pInfo->pState, &key);
5
54liuyao 已提交
1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
  return true;
}

static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int32_t numOfOutput, SSDataBlock* pBlock,
                            SArray* pUpWins, SHashObj* pUpdatedMap) {
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  TSKEY*           startTsCols = (TSKEY*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           endTsCols = (TSKEY*)pEndTsCol->pData;
  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*        pGpDatas = (uint64_t*)pGpCol->pData;
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
1405
    while (win.skey <= endTsCols[i]) {
5
54liuyao 已提交
1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419
      uint64_t winGpId = pGpDatas[i];
      bool     res = doDeleteWindow(pOperator, win.skey, winGpId, numOfOutput);
      SWinKey  winRes = {.ts = win.skey, .groupId = winGpId};
      if (pUpWins && res) {
        taosArrayPush(pUpWins, &winRes);
      }
      if (pUpdatedMap) {
        taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
    }
  }
}

1420 1421 1422 1423
bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) {
  size_t bytes = sizeof(TSKEY);
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId);
  SResultRowPosition* p1 =
1424
      (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1425 1426 1427 1428
  if (!p1) {
    // window has been closed
    return false;
  }
1429
  tSimpleHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1430 1431 1432
  return true;
}

5
54liuyao 已提交
1433 1434
static void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, STimeWindowAggSupp* pTwSup, SSDataBlock* pBlock,
                                          SArray* pDelWins, SInterval* pInterval, SHashObj* pUpdatedMap) {
1435
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
1436
  TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
1437 1438
  SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           tsEnds = (TSKEY*)pEndCol->pData;
1439
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
1440
  uint64_t*        groupIds = (uint64_t*)pGroupCol->pData;
5
54liuyao 已提交
1441
  int64_t          numOfWin = tSimpleHashGetSize(pAggSup->pResultRowHashTable);
1442
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
5
54liuyao 已提交
1443 1444
    TSKEY          startTs = TMAX(tsStarts[i], pTwSup->minTs);
    TSKEY          endTs = TMIN(tsEnds[i], pTwSup->maxTs);
1445 1446
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
5
54liuyao 已提交
1447
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTs, pInterval, TSDB_ORDER_ASC);
1448 1449 1450 1451 1452 1453 1454 1455 1456 1457
    do {
      doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
      SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]};
      if (pDelWins) {
        taosArrayPush(pDelWins, &winRes);
      }
      if (pUpdatedMap) {
        taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
5
54liuyao 已提交
1458
    } while (win.skey <= endTs);
1459 1460 1461
  }
}

1462 1463
static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput,
                           SSDataBlock* pBlock, SArray* pUpWins) {
1464 1465 1466 1467
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  TSKEY*           startTsCols = (TSKEY*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           endTsCols = (TSKEY*)pEndTsCol->pData;
1468 1469
  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*        pGpDatas = (uint64_t*)pGpCol->pData;
1470 1471 1472 1473 1474
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
    while (win.ekey <= endTsCols[i]) {
1475
      uint64_t winGpId = pGpDatas[i];
L
Liu Jicong 已提交
1476
      bool     res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput);
1477
      if (pUpWins && res) {
H
Haojun Liao 已提交
1478
        SWinKey winRes = {.ts = win.skey, .groupId = winGpId};
1479 1480 1481
        taosArrayPush(pUpWins, &winRes);
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
5
54liuyao 已提交
1482
    }
5
54liuyao 已提交
1483 1484
  }
}
1485

1486 1487 1488 1489 1490 1491
static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) {
  void*   pIte = NULL;
  size_t  keyLen = 0;
  int32_t iter = 0;
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    void*    key = tSimpleHashGetKey(pIte, &keyLen);
5
54liuyao 已提交
1492 1493
    uint64_t groupId = *(uint64_t*)key;
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
1494
    TSKEY               ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1495
    SResultRowPosition* pPos = (SResultRowPosition*)pIte;
5
54liuyao 已提交
1496
    int32_t             code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
5
54liuyao 已提交
1497 1498 1499 1500 1501 1502 1503
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
1504 1505 1506
static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval,
                                         SHashObj* pPullDataMap, SHashObj* closeWins, SOperatorInfo* pOperator) {
  qDebug("===stream===close interval window");
1507 1508 1509 1510
  void*                        pIte = NULL;
  size_t                       keyLen = 0;
  int32_t                      iter = 0;
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
5
54liuyao 已提交
1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538
  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
    void*       key = tSimpleHashGetKey(pIte, &keyLen);
    SWinKey*    pWinKey = (SWinKey*)key;
    void*       chIds = taosHashGet(pPullDataMap, pWinKey, sizeof(SWinKey));
    STimeWindow win = {
        .skey = pWinKey->ts,
        .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1,
    };
    if (isCloseWindow(&win, pTwSup)) {
      if (chIds && pPullDataMap) {
        SArray* chAy = *(SArray**)chIds;
        int32_t size = taosArrayGetSize(chAy);
        qDebug("===stream===window %" PRId64 " wait child size:%d", pWinKey->ts, size);
        for (int32_t i = 0; i < size; i++) {
          qDebug("===stream===window %" PRId64 " wait child id:%d", pWinKey->ts, *(int32_t*)taosArrayGet(chAy, i));
        }
        continue;
      } else if (pPullDataMap) {
        qDebug("===stream===close window %" PRId64, pWinKey->ts);
      }

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
        int32_t code = saveWinResultInfo(pWinKey->ts, pWinKey->groupId, closeWins);
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
      }
      tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter);
1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583
    }
  }
  return TSDB_CODE_SUCCESS;
}

STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
  return w;
}

static void deleteIntervalDiscBuf(SStreamState* pState, SHashObj* pPullDataMap, TSKEY mark, SInterval* pInterval,
                                  SWinKey* key) {
  STimeWindow tw = getFinalTimeWindow(key->ts, pInterval);
  SWinKey     next = {0};
  while (tw.ekey < mark) {
    SStreamStateCur* pCur = streamStateSeekKeyNext(pState, key);
    int32_t          code = streamStateGetKVByCur(pCur, &next, NULL, 0);
    streamStateFreeCur(pCur);

    void* chIds = taosHashGet(pPullDataMap, key, sizeof(SWinKey));
    if (chIds && pPullDataMap) {
      SArray* chAy = *(SArray**)chIds;
      int32_t size = taosArrayGetSize(chAy);
      qDebug("===stream===window %" PRId64 " wait child size:%d", key->ts, size);
      for (int32_t i = 0; i < size; i++) {
        qDebug("===stream===window %" PRId64 " wait child id:%d", key->ts, *(int32_t*)taosArrayGet(chAy, i));
      }
      break;
    }
    qDebug("===stream===delete window %" PRId64, key->ts);
    int32_t codeDel = streamStateDel(pState, key);
    if (codeDel != TSDB_CODE_SUCCESS) {
      code = streamStateGetFirst(pState, key);
      if (code != TSDB_CODE_SUCCESS) {
        qDebug("===stream===stream state first key: empty-empty");
        return;
      }
      continue;
    }
    if (code == TSDB_CODE_SUCCESS) {
      *key = next;
      tw = getFinalTimeWindow(key->ts, pInterval);
    }
  }
5
54liuyao 已提交
1584

1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598
  if (qDebugFlag & DEBUG_DEBUG) {
    SStreamStateCur* pCur = streamStateGetCur(pState, key);
    int32_t          code = streamStateCurPrev(pState, pCur);
    if (code == TSDB_CODE_SUCCESS) {
      SWinKey tmpKey = {0};
      code = streamStateGetKVByCur(pCur, &tmpKey, NULL, 0);
      if (code == TSDB_CODE_SUCCESS) {
        STimeWindow tw = getFinalTimeWindow(tmpKey.ts, pInterval);
        qDebug("===stream===error stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey,
               tw.ekey, tmpKey.groupId, mark);
      } else {
        STimeWindow tw = getFinalTimeWindow(key->ts, pInterval);
        qDebug("===stream===stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey, tw.ekey,
               key->groupId, mark);
5
54liuyao 已提交
1599
      }
1600 1601 1602 1603
    } else {
      STimeWindow tw = getFinalTimeWindow(key->ts, pInterval);
      qDebug("===stream===stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey, tw.ekey,
             key->groupId, mark);
5
54liuyao 已提交
1604
    }
1605
    streamStateFreeCur(pCur);
5
54liuyao 已提交
1606 1607 1608
  }
}

1609
static void closeChildIntervalWindow(SOperatorInfo* pOperator, SArray* pChildren, TSKEY maxTs) {
5
54liuyao 已提交
1610 1611
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
1612 1613
    SOperatorInfo*               pChildOp = taosArrayGetP(pChildren, i);
    SStreamIntervalOperatorInfo* pChInfo = pChildOp->info;
5
54liuyao 已提交
1614 1615
    ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
1616 1617
    closeStreamIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL,
                              pOperator);
1618 1619 1620 1621 1622 1623 1624
  }
}

static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) {
  int32_t size = taosArrayGetSize(pageIds);
  for (int32_t i = 0; i < size; i++) {
    int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i);
5
54liuyao 已提交
1625
    // SFilePage* bufPage = getBufPage(pDiskBuf, pageId);
1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639
    // dBufSetBufPageRecycled(pDiskBuf, bufPage);
  }
  taosArrayClear(pageIds);
}

static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlock) {
  blockDataCleanup(pBlock);
  int32_t size = taosArrayGetSize(pWins);
  if (*index == size) {
    *index = 0;
    taosArrayClear(pWins);
    return;
  }
  blockDataEnsureCapacity(pBlock, size - *index);
1640
  uint64_t uid = 0;
1641
  for (int32_t i = *index; i < size; i++) {
H
Haojun Liao 已提交
1642
    SWinKey* pWin = taosArrayGet(pWins, i);
1643
    appendOneRow(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId);
1644
    (*index)++;
5
54liuyao 已提交
1645 1646 1647
  }
}

1648
static void destroyStateWindowOperatorInfo(void* param) {
1649
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
1650
  cleanupBasicInfo(&pInfo->binfo);
1651
  taosMemoryFreeClear(pInfo->stateKey.pData);
1652

D
dapan1121 已提交
1653
  taosMemoryFreeClear(param);
1654 1655
}

H
Haojun Liao 已提交
1656
static void freeItem(void* param) {
L
Liu Jicong 已提交
1657
  SGroupKeys* pKey = (SGroupKeys*)param;
H
Haojun Liao 已提交
1658 1659 1660
  taosMemoryFree(pKey->pData);
}

1661
void destroyIntervalOperatorInfo(void* param) {
1662
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
1663
  cleanupBasicInfo(&pInfo->binfo);
1664
  cleanupAggSup(&pInfo->aggSup);
1665 1666 1667 1668
  cleanupExprSupp(&pInfo->scalarSupp);

  tdListFree(pInfo->binfo.resultRowInfo.openWindow);

H
Haojun Liao 已提交
1669 1670 1671 1672
  pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols);
  taosArrayDestroyEx(pInfo->pPrevValues, freeItem);

  pInfo->pPrevValues = NULL;
1673

H
Haojun Liao 已提交
1674 1675
  cleanupGroupResInfo(&pInfo->groupResInfo);
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
D
dapan1121 已提交
1676
  taosMemoryFreeClear(param);
1677 1678
}

1679
void destroyStreamFinalIntervalOperatorInfo(void* param) {
1680
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;
1681
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
1682
  cleanupAggSup(&pInfo->aggSup);
L
Liu Jicong 已提交
1683
  // it should be empty.
5
54liuyao 已提交
1684 1685 1686
  taosHashCleanup(pInfo->pPullDataMap);
  taosArrayDestroy(pInfo->pPullWins);
  blockDataDestroy(pInfo->pPullDataRes);
H
Haojun Liao 已提交
1687
  blockDataDestroy(pInfo->pUpdateRes);
L
Liu Jicong 已提交
1688 1689
  taosArrayDestroy(pInfo->pDelWins);
  blockDataDestroy(pInfo->pDelRes);
1690
  taosMemoryFreeClear(pInfo->pState);
5
54liuyao 已提交
1691

1692 1693 1694 1695
  if (pInfo->pChildren) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
1696
      destroyStreamFinalIntervalOperatorInfo(pChildOp->info);
L
Liu Jicong 已提交
1697 1698
      taosMemoryFree(pChildOp->pDownstream);
      cleanupExprSupp(&pChildOp->exprSupp);
1699 1700
      taosMemoryFreeClear(pChildOp);
    }
L
Liu Jicong 已提交
1701
    taosArrayDestroy(pInfo->pChildren);
1702
  }
1703
  nodesDestroyNode((SNode*)pInfo->pPhyNode);
5
54liuyao 已提交
1704
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
1705
  cleanupGroupResInfo(&pInfo->groupResInfo);
1706

D
dapan1121 已提交
1707
  taosMemoryFreeClear(param);
5
54liuyao 已提交
1708 1709
}

1710
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
5
54liuyao 已提交
1711
  for (int32_t i = 0; i < numOfCols; i++) {
5
54liuyao 已提交
1712
    if (fmIsUserDefinedFunc(pFCtx[i].functionId) || !fmIsInvertible(pFCtx[i].functionId)) {
5
54liuyao 已提交
1713 1714 1715 1716 1717 1718
      return false;
    }
  }
  return true;
}

1719
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
1720 1721
  // the primary timestamp column
  bool needed = false;
1722 1723
  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
1724

X
Xiaoyu Wang 已提交
1725
  {  // ts column
1726 1727
    SColumn c = {0};
    c.colId = 1;
1728
    c.slotId = pInfo->primaryTsIndex;
1729 1730
    c.type = TSDB_DATA_TYPE_TIMESTAMP;
    c.bytes = sizeof(int64_t);
1731
    taosArrayPush(pInfo->pInterpCols, &c);
1732 1733

    SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1734 1735 1736 1737
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = true;  // to denote no value is assigned yet
    key.pData = taosMemoryCalloc(1, c.bytes);
1738
    taosArrayPush(pInfo->pPrevValues, &key);
1739 1740
  }

X
Xiaoyu Wang 已提交
1741
  for (int32_t i = 0; i < numOfCols; ++i) {
1742 1743
    SExprInfo* pExpr = pCtx[i].pExpr;

H
Haojun Liao 已提交
1744
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1745 1746 1747
      SFunctParam* pParam = &pExpr->base.pParam[0];

      SColumn c = *pParam->pCol;
1748
      taosArrayPush(pInfo->pInterpCols, &c);
1749 1750 1751
      needed = true;

      SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1752 1753
      key.bytes = c.bytes;
      key.type = c.type;
1754
      key.isNull = false;
X
Xiaoyu Wang 已提交
1755
      key.pData = taosMemoryCalloc(1, c.bytes);
1756
      taosArrayPush(pInfo->pPrevValues, &key);
1757 1758 1759 1760 1761 1762
    }
  }

  return needed;
}

L
Liu Jicong 已提交
1763
void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval,
5
54liuyao 已提交
1764
                            STimeWindowAggSupp* pTwSup) {
1765
  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
5
54liuyao 已提交
1766
    initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, pTwSup);
1767 1768
    return;
  }
5
54liuyao 已提交
1769
  SStreamScanInfo* pScanInfo = downstream->info;
1770 1771
  pScanInfo->windowSup.parentType = type;
  pScanInfo->windowSup.pIntervalAggSup = pSup;
5
54liuyao 已提交
1772
  pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, pTwSup->waterMark);
1773
  pScanInfo->interval = *pInterval;
5
54liuyao 已提交
1774
  pScanInfo->twAggSup = *pTwSup;
5
54liuyao 已提交
1775 1776
}

H
Haojun Liao 已提交
1777 1778 1779 1780 1781 1782
void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
  for (int32_t i = 0; i < numOfExpr; i++) {
    pCtx[i].isStream = true;
  }
}

1783 1784
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                          SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
L
Liu Jicong 已提交
1785 1786
                                          STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode,
                                          SExecTaskInfo* pTaskInfo, bool isStream) {
1787
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1788
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1789 1790 1791 1792
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1793
  pOperator->pTaskInfo = pTaskInfo;
L
Liu Jicong 已提交
1794
  pInfo->win = pTaskInfo->window;
1795 1796
  pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
  pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
L
Liu Jicong 已提交
1797
  pInfo->interval = *pInterval;
L
Liu Jicong 已提交
1798
  pInfo->execModel = pTaskInfo->execModel;
L
Liu Jicong 已提交
1799
  pInfo->twAggSup = *pTwAggSupp;
1800
  pInfo->pCondition = pPhyNode->window.node.pConditions;
1801
  pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock;
1802 1803 1804 1805

  if (pPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
L
Liu Jicong 已提交
1806
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
1807 1808 1809 1810
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
1811

1812
  pInfo->primaryTsIndex = primaryTsSlotId;
1813 1814
  SExprSupp* pSup = &pOperator->exprSupp;

1815
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
1816
  initResultSizeInfo(&pOperator->resultInfo, 4096);
1817

1818
  int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
1819 1820 1821 1822
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

1823 1824
  initBasicInfo(&pInfo->binfo, pResBlock);

1825 1826
  if (isStream) {
    ASSERT(numOfCols > 0);
H
Haojun Liao 已提交
1827
    initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
1828
  }
1829

1830
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
1831

1832
  pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, pInfo);
1833
  if (pInfo->timeWindowInterpo) {
1834
    pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
H
Haojun Liao 已提交
1835 1836 1837
    if (pInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
1838
  }
1839

1840
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1841

X
Xiaoyu Wang 已提交
1842 1843 1844 1845 1846
  pOperator->name = "TimeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
1847

1848
  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL,
1849 1850 1851 1852 1853 1854 1855 1856 1857
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1858
_error:
1859
  destroyIntervalOperatorInfo(pInfo);
1860 1861 1862 1863 1864
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

1865
// todo handle multiple timeline cases. assume no timeline interweaving
1866 1867
static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1868
  SExprSupp*     pSup = &pOperator->exprSupp;
1869

1870
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1871 1872

  bool    masterScan = true;
1873
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888
  int64_t gid = pBlock->info.groupId;

  int64_t gap = pInfo->gap;

  if (!pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->winSup.prevTs = INT64_MIN;
  }

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

  // In case of ascending or descending order scan data, only one time window needs to be kepted for each table.
  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
1889 1890 1891
    if (gid != pRowSup->groupId || pInfo->winSup.prevTs == INT64_MIN) {
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
H
Haojun Liao 已提交
1892 1893
    } else if (((tsList[j] - pRowSup->prevTs >= 0) && (tsList[j] - pRowSup->prevTs <= gap)) ||
               ((pRowSup->prevTs - tsList[j] >= 0) && (pRowSup->prevTs - tsList[j] <= gap))) {
1894
      // The gap is less than the threshold, so it belongs to current session window that has been opened already.
1895
      doKeepTuple(pRowSup, tsList[j], gid);
1896 1897 1898 1899 1900 1901 1902 1903 1904 1905
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1906 1907
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1908
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1909
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1910 1911 1912 1913
      }

      // pInfo->numOfRows data belong to the current session window
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1914 1915
      doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                       pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
1916 1917

      // here we start a new session window
1918 1919
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1920 1921 1922 1923 1924
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1925 1926
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1927
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
1928
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
1929 1930 1931
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
H
Haojun Liao 已提交
1932 1933
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
                   pBlock->info.rows, numOfOutput);
1934 1935
}

1936
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
1937 1938 1939 1940 1941 1942
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*          pBInfo = &pInfo->binfo;
1943
  SExprSupp*               pSup = &pOperator->exprSupp;
1944 1945

  if (pOperator->status == OP_RES_TO_RETURN) {
1946
    while (1) {
1947
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1948
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1949

1950
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1951 1952 1953 1954
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
1955

1956 1957 1958 1959 1960
      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1961
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1962 1963
  }

1964 1965 1966
  int64_t st = taosGetTimestampUs();
  int32_t order = TSDB_ORDER_ASC;

1967 1968 1969
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1970
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1971 1972 1973 1974 1975
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1976
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1977 1978
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1979 1980 1981
    doSessionWindowAggImpl(pOperator, pInfo, pBlock);
  }

1982 1983
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;

1984 1985 1986
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;

1987
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1988
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1989
  while (1) {
1990
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1991
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1992

1993
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1994 1995 1996 1997
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
1998

1999 2000 2001 2002 2003
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
2004
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
2005 2006
}

2007
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
H
Haojun Liao 已提交
2008
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2009 2010
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
H
Haojun Liao 已提交
2011 2012 2013 2014 2015 2016

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);

      pkey->isNull = false;
2017
      char* val = colDataGetData(pColInfoData, rowIndex);
2018 2019 2020 2021 2022
      if (!IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, pkey->bytes);
      } else {
        memcpy(pkey->pData, val, varDataLen(val));
      }
H
Haojun Liao 已提交
2023 2024
    }
  }
2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039

  pSliceInfo->isPrevRowSet = true;
}

static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i);

      pkey->isNull = false;
      char* val = colDataGetData(pColInfoData, rowIndex);
2040 2041 2042 2043 2044
      if (!IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, pkey->bytes);
      } else {
        memcpy(pkey->pData, val, varDataLen(val));
      }
2045 2046 2047 2048
    }
  }

  pSliceInfo->isNextRowSet = true;
H
Haojun Liao 已提交
2049 2050
}

2051 2052
static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex,
                             bool isLastRow) {
2053
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
dengyihao's avatar
dengyihao 已提交
2054
  bool    fillLastPoint = pSliceInfo->fillLastPoint;
2055 2056
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
2057 2058
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
    SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i);
2059 2060 2061

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077
      if (isLastRow) {
        pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
      } else if (fillLastPoint) {
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
      } else {
        pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1);

        char* val;
        val = colDataGetData(pColInfoData, rowIndex);
        memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes);
        val = colDataGetData(pColInfoData, rowIndex + 1);
        memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes);
      }
2078 2079 2080 2081

      pLinearInfo->hasNull = false;
    } else {
      pLinearInfo->hasNull = true;
2082 2083 2084
    }
  }

2085
  pSliceInfo->fillLastPoint = isLastRow;
2086 2087
}

dengyihao's avatar
dengyihao 已提交
2088
static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) {
2089
  int32_t rows = pResBlock->info.rows;
2090
  blockDataEnsureCapacity(pResBlock, rows + 1);
2091 2092 2093
  // todo set the correct primary timestamp column

  // output the result
2094
  bool hasInterp = true;
2095 2096 2097
  for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
    SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];

2098
    int32_t          dstSlot = pExprInfo->base.resSchema.slotId;
2099 2100
    SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

2101 2102 2103 2104 2105
    if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
      colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false);
      continue;
    }

G
Ganlin Zhao 已提交
2106
    int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
2107
    // SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
2108
    switch (pSliceInfo->fillType) {
2109
      case TSDB_FILL_NULL: {
2110 2111
        colDataAppendNULL(pDst, rows);
        break;
2112
      }
2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129

      case TSDB_FILL_SET_VALUE: {
        SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal;

        if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
          float v = 0;
          GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
          double v = 0;
          GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
          int64_t v = 0;
          GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
          colDataAppend(pDst, rows, (char*)&v, false);
        }
2130 2131
        break;
      }
2132

2133
      case TSDB_FILL_LINEAR: {
2134
        SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, srcSlot);
2135

dengyihao's avatar
dengyihao 已提交
2136 2137
        SPoint start = pLinearInfo->start;
        SPoint end = pLinearInfo->end;
2138 2139 2140
        SPoint current = {.key = pSliceInfo->current};
        current.val = taosMemoryCalloc(pLinearInfo->bytes, 1);

2141 2142
        // before interp range, do not fill
        if (start.key == INT64_MIN || end.key == INT64_MAX) {
2143
          hasInterp = false;
2144 2145 2146
          break;
        }

2147 2148 2149 2150
        if (pLinearInfo->hasNull) {
          colDataAppendNULL(pDst, rows);
        } else {
          taosGetLinearInterpolationVal(&current, pLinearInfo->type, &start, &end, pLinearInfo->type);
dengyihao's avatar
dengyihao 已提交
2151
          colDataAppend(pDst, rows, (char*)current.val, false);
2152 2153
        }

2154
        taosMemoryFree(current.val);
2155
        break;
2156
      }
2157
      case TSDB_FILL_PREV: {
2158
        if (!pSliceInfo->isPrevRowSet) {
2159
          hasInterp = false;
2160 2161 2162
          break;
        }

2163 2164
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
        colDataAppend(pDst, rows, pkey->pData, false);
2165 2166
        break;
      }
2167 2168

      case TSDB_FILL_NEXT: {
2169
        if (!pSliceInfo->isNextRowSet) {
2170
          hasInterp = false;
2171 2172 2173
          break;
        }

2174 2175
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot);
        colDataAppend(pDst, rows, pkey->pData, false);
2176 2177
        break;
      }
2178 2179 2180 2181 2182 2183

      case TSDB_FILL_NONE:
      default:
        break;
    }
  }
2184 2185 2186 2187

  if (hasInterp) {
    pResBlock->info.rows += 1;
  }
2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199
}

static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pPrevRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pPrevRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

2200
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pPrevRow, &key);
  }

2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240
  pInfo->isPrevRowSet = false;

  return TSDB_CODE_SUCCESS;
}

static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pNextRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pNextRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pNextRow, &key);
  }

  pInfo->isNextRowSet = false;

2241 2242 2243
  return TSDB_CODE_SUCCESS;
}

2244 2245 2246 2247 2248 2249
static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pLinearInfo != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo));
2250
  if (pInfo->pLinearInfo == NULL) {
2251 2252 2253 2254 2255 2256 2257
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

2258 2259
    SFillLinearInfo linearInfo = {0};
    linearInfo.start.key = INT64_MIN;
dengyihao's avatar
dengyihao 已提交
2260
    linearInfo.end.key = INT64_MAX;
2261
    linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes);
dengyihao's avatar
dengyihao 已提交
2262 2263 2264
    linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes);
    linearInfo.hasNull = false;
    linearInfo.type = pColInfo->info.type;
2265 2266
    linearInfo.bytes = pColInfo->info.bytes;
    taosArrayPush(pInfo->pLinearInfo, &linearInfo);
2267 2268
  }

2269 2270
  pInfo->fillLastPoint = false;

2271 2272 2273
  return TSDB_CODE_SUCCESS;
}

2274 2275 2276 2277
static bool needToFillLastPoint(STimeSliceOperatorInfo* pSliceInfo) {
  return (pSliceInfo->fillLastPoint == true && pSliceInfo->fillType == TSDB_FILL_LINEAR);
}

2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294
static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  int32_t code;
  code = initPrevRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initNextRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initFillLinearInfo(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

2295 2296 2297
  return TSDB_CODE_SUCCESS;
}

H
Haojun Liao 已提交
2298
static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
2299 2300 2301 2302
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

2303 2304
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

2305
  STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
2306 2307
  SSDataBlock*            pResBlock = pSliceInfo->pRes;
  SExprSupp*              pSup = &pOperator->exprSupp;
H
Haojun Liao 已提交
2308

2309 2310
  int32_t        order = TSDB_ORDER_ASC;
  SInterval*     pInterval = &pSliceInfo->interval;
2311 2312
  SOperatorInfo* downstream = pOperator->pDownstream[0];

2313 2314
  blockDataCleanup(pResBlock);

2315
  while (1) {
2316
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
2317 2318 2319 2320
    if (pBlock == NULL) {
      break;
    }

2321
    int32_t code = initKeeperInfo(pSliceInfo, pBlock);
2322
    if (code != TSDB_CODE_SUCCESS) {
2323
      T_LONG_JMP(pTaskInfo->env, code);
2324 2325
    }

2326
    // the pDataBlock are always the same one, no need to call this again
2327
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
H
Haojun Liao 已提交
2328

2329
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
2330 2331
    for (int32_t i = 0; i < pBlock->info.rows; ++i) {
      int64_t ts = *(int64_t*)colDataGetData(pTsCol, i);
H
Haojun Liao 已提交
2332

dengyihao's avatar
dengyihao 已提交
2333
      if (i == 0 && needToFillLastPoint(pSliceInfo)) {  // first row in current block
2334 2335 2336 2337 2338 2339 2340 2341 2342
        doKeepLinearInfo(pSliceInfo, pBlock, i, false);
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
          genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
        }
2343
      }
2344

2345 2346 2347
      if (pSliceInfo->current > pSliceInfo->win.ekey) {
        doSetOperatorCompleted(pOperator);
        break;
2348 2349
      }

H
Haojun Liao 已提交
2350
      if (ts == pSliceInfo->current) {
2351
        for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
2352
          SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
H
Haojun Liao 已提交
2353

2354
          int32_t          dstSlot = pExprInfo->base.resSchema.slotId;
2355
          SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
H
Haojun Liao 已提交
2356

2357
          if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
2358
            colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false);
2359
          } else {
2360
            int32_t          srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
2361
            SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
2362

2363 2364 2365 2366 2367 2368 2369 2370
            if (colDataIsNull_s(pSrc, i)) {
              colDataAppendNULL(pDst, pResBlock->info.rows);
              continue;
            }

            char* v = colDataGetData(pSrc, i);
            colDataAppend(pDst, pResBlock->info.rows, v, false);
          }
H
Haojun Liao 已提交
2371 2372
        }

2373
        pResBlock->info.rows += 1;
2374
        doKeepPrevRows(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2375

2376 2377
        // for linear interpolation, always fill value between this and next points;
        // if its the first point in data block, also fill values between previous(if there's any) and this point;
2378 2379
        // if its the last point in data block, no need to fill, but reserve this point as the start value and do
        // the interpolation when processing next data block.
2380
        if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2381 2382
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2383
          if (i < pBlock->info.rows - 1) {
2384
            doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2385 2386 2387
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2388
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2389 2390
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2391 2392 2393 2394
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }
2395

2396 2397 2398 2399 2400
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
            }
dengyihao's avatar
dengyihao 已提交
2401 2402
          } else {  // it is the last row of current block
            // store ts value as start, and calculate interp value when processing next block
2403
            doKeepLinearInfo(pSliceInfo, pBlock, i, true);
2404
          }
dengyihao's avatar
dengyihao 已提交
2405
        } else {  // non-linear interpolation
2406 2407 2408 2409 2410 2411
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (pSliceInfo->current > pSliceInfo->win.ekey) {
            doSetOperatorCompleted(pOperator);
            break;
          }
2412

2413 2414 2415
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
2416
        }
H
Haojun Liao 已提交
2417
      } else if (ts < pSliceInfo->current) {
2418
        // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate
2419 2420
        doKeepPrevRows(pSliceInfo, pBlock, i);

2421
        if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2422
          // no need to increate pSliceInfo->current here
dengyihao's avatar
dengyihao 已提交
2423
          // pSliceInfo->current =
2424 2425
          //    taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (i < pBlock->info.rows - 1) {
2426
            doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2427 2428 2429
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2430
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2431 2432
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2433 2434 2435 2436 2437 2438 2439
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }

              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
H
Haojun Liao 已提交
2440 2441
                break;
              }
H
Haojun Liao 已提交
2442
            }
2443 2444 2445
          } else {
            // store ts value as start, and calculate interp value when processing next block
            doKeepLinearInfo(pSliceInfo, pBlock, i, true);
2446
          }
dengyihao's avatar
dengyihao 已提交
2447
        } else {  // non-linear interpolation
2448 2449 2450 2451 2452 2453
          if (i < pBlock->info.rows - 1) {
            // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
            doKeepNextRows(pSliceInfo, pBlock, i + 1);
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2454
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2455 2456
                pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                  pInterval->precision);
2457 2458 2459 2460
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }
2461

2462 2463 2464 2465 2466 2467
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
            } else {
              // ignore current row, and do nothing
H
Haojun Liao 已提交
2468
            }
2469 2470
          } else {  // it is the last row of current block
            doKeepPrevRows(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2471
          }
2472 2473
        }
      } else {  // ts > pSliceInfo->current
2474
        // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
2475 2476
        doKeepNextRows(pSliceInfo, pBlock, i);

2477
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
2478
          genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
2479 2480
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2481 2482 2483
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
2484 2485
        }

2486 2487 2488 2489 2490
        // add current row if timestamp match
        if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
          for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
            SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];

2491
            int32_t          dstSlot = pExprInfo->base.resSchema.slotId;
2492 2493
            SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

2494
            if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
2495
              colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false);
2496
            } else {
2497
              int32_t          srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
2498 2499 2500 2501 2502 2503 2504 2505 2506 2507
              SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);

              if (colDataIsNull_s(pSrc, i)) {
                colDataAppendNULL(pDst, pResBlock->info.rows);
                continue;
              }

              char* v = colDataGetData(pSrc, i);
              colDataAppend(pDst, pResBlock->info.rows, v, false);
            }
2508 2509 2510 2511 2512
          }

          pResBlock->info.rows += 1;
          doKeepPrevRows(pSliceInfo, pBlock, i);

2513 2514 2515 2516
          if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
            pSliceInfo->current =
                taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
            if (i < pBlock->info.rows - 1) {
2517
              doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2518 2519 2520
              int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
              if (nextTs > pSliceInfo->current) {
                while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2521
                  genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
dengyihao's avatar
dengyihao 已提交
2522 2523
                  pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit,
                                                    pInterval->precision);
2524 2525 2526 2527
                  if (pResBlock->info.rows >= pResBlock->info.capacity) {
                    break;
                  }
                }
2528

2529 2530 2531 2532 2533
                if (pSliceInfo->current > pSliceInfo->win.ekey) {
                  doSetOperatorCompleted(pOperator);
                  break;
                }
              }
2534 2535 2536
            } else {  // it is the last row of current block
              // store ts value as start, and calculate interp value when processing next block
              doKeepLinearInfo(pSliceInfo, pBlock, i, true);
2537
            }
dengyihao's avatar
dengyihao 已提交
2538
          } else {  // non-linear interpolation
2539 2540 2541 2542 2543 2544
            pSliceInfo->current =
                taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);

            if (pResBlock->info.rows >= pResBlock->info.capacity) {
              break;
            }
2545 2546 2547
          }
        }

2548 2549 2550
        if (pSliceInfo->current > pSliceInfo->win.ekey) {
          doSetOperatorCompleted(pOperator);
          break;
H
Haojun Liao 已提交
2551 2552 2553
        }
      }
    }
2554
  }
2555

2556 2557
  // check if need to interpolate after last datablock
  // except for fill(next), fill(linear)
dengyihao's avatar
dengyihao 已提交
2558 2559
  while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT &&
         pSliceInfo->fillType != TSDB_FILL_LINEAR) {
2560 2561 2562 2563 2564
    genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
    pSliceInfo->current =
        taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
    if (pResBlock->info.rows >= pResBlock->info.capacity) {
      break;
2565
    }
2566 2567 2568 2569
  }

  // restore the value
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
2570
  if (pResBlock->info.rows == 0) {
2571 2572 2573
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
2574 2575 2576
  return pResBlock->info.rows == 0 ? NULL : pResBlock;
}

2577
void destroyTimeSliceOperatorInfo(void* param) {
2578 2579 2580 2581 2582 2583 2584 2585
  STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param;

  pInfo->pRes = blockDataDestroy(pInfo->pRes);

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) {
    SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i);
    taosMemoryFree(pKey->pData);
  }
2586
  taosArrayDestroy(pInfo->pPrevRow);
2587 2588 2589 2590 2591 2592

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pNextRow); ++i) {
    SGroupKeys* pKey = taosArrayGet(pInfo->pNextRow, i);
    taosMemoryFree(pKey->pData);
  }
  taosArrayDestroy(pInfo->pNextRow);
2593 2594 2595 2596 2597 2598

  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pLinearInfo); ++i) {
    SFillLinearInfo* pKey = taosArrayGet(pInfo->pLinearInfo, i);
    taosMemoryFree(pKey->start.val);
    taosMemoryFree(pKey->end.val);
  }
2599 2600
  taosArrayDestroy(pInfo->pLinearInfo);

2601
  taosMemoryFree(pInfo->pFillColInfo);
2602 2603 2604
  taosMemoryFreeClear(param);
}

2605
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
2606 2607 2608 2609 2610 2611
  STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pOperator == NULL || pInfo == NULL) {
    goto _error;
  }

2612
  SInterpFuncPhysiNode* pInterpPhyNode = (SInterpFuncPhysiNode*)pPhyNode;
2613
  SExprSupp*            pSup = &pOperator->exprSupp;
2614

2615
  int32_t    numOfExprs = 0;
2616
  SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
2617
  int32_t    code = initExprSupp(pSup, pExprInfo, numOfExprs);
H
Haojun Liao 已提交
2618 2619 2620 2621
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2622
  if (pInterpPhyNode->pExprs != NULL) {
2623
    int32_t    num = 0;
2624 2625 2626 2627 2628 2629 2630 2631 2632
    SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

  pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries);
  pInfo->fillType = convertFillType(pInterpPhyNode->fillMode);
2633
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2634

H
Haojun Liao 已提交
2635
  pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, NULL, 0, (SNodeListNode*)pInterpPhyNode->pFillValues);
2636
  pInfo->pLinearInfo = NULL;
L
Liu Jicong 已提交
2637 2638
  pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  pInfo->win = pInterpPhyNode->timeRange;
2639
  pInfo->interval.interval = pInterpPhyNode->interval;
L
Liu Jicong 已提交
2640
  pInfo->current = pInfo->win.skey;
H
Haojun Liao 已提交
2641

2642 2643 2644 2645
  STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info;
  pScanInfo->cond.twindows = pInfo->win;
  pScanInfo->cond.type = TIMEWINDOW_RANGE_EXTERNAL;

2646
  pOperator->name = "TimeSliceOperator";
2647
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC;
2648 2649 2650 2651
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
  pOperator->pTaskInfo = pTaskInfo;
2652

2653
  pOperator->fpSet =
2654
      createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyTimeSliceOperatorInfo, NULL, NULL, NULL);
2655

2656 2657
  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);

H
Haojun Liao 已提交
2658
  code = appendDownstream(pOperator, &downstream, 1);
2659 2660
  return pOperator;

L
Liu Jicong 已提交
2661
_error:
2662 2663 2664 2665 2666 2667
  taosMemoryFree(pInfo);
  taosMemoryFree(pOperator);
  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
  return NULL;
}

2668 2669
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
                                             SExecTaskInfo* pTaskInfo) {
2670 2671 2672 2673 2674 2675
  SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

L
Liu Jicong 已提交
2676
  int32_t      num = 0;
2677 2678 2679 2680 2681 2682 2683
  SExprInfo*   pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc);
  int32_t      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;

  SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
2684 2685 2686
  pInfo->stateKey.type = pInfo->stateCol.type;
  pInfo->stateKey.bytes = pInfo->stateCol.bytes;
  pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
2687
  pInfo->pCondition = pStateNode->window.node.pConditions;
2688 2689 2690 2691
  if (pInfo->stateKey.pData == NULL) {
    goto _error;
  }

2692 2693
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

2694
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2695
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
2696 2697 2698 2699
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2700
  initBasicInfo(&pInfo->binfo, pResBlock);
2701
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2702

L
Liu Jicong 已提交
2703 2704 2705
  pInfo->twAggSup =
      (STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType};
  ;
2706 2707
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

X
Xiaoyu Wang 已提交
2708 2709
  pInfo->tsSlotId = tsSlotId;
  pOperator->name = "StateWindowOperator";
2710
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE;
X
Xiaoyu Wang 已提交
2711 2712 2713 2714
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
2715 2716 2717 2718

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL,
                                         destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

2719 2720 2721 2722 2723
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2724 2725
  return pOperator;

L
Liu Jicong 已提交
2726
_error:
2727 2728 2729
  destroyStateWindowOperatorInfo(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
2730 2731 2732
  return NULL;
}

2733
void destroySWindowOperatorInfo(void* param) {
2734
  SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
2735 2736 2737
  if (pInfo == NULL) {
    return;
  }
2738

2739
  cleanupBasicInfo(&pInfo->binfo);
H
Haojun Liao 已提交
2740 2741 2742 2743
  colDataDestroy(&pInfo->twAggSup.timeWindowData);

  cleanupAggSup(&pInfo->aggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
D
dapan1121 已提交
2744
  taosMemoryFreeClear(param);
2745 2746
}

H
Haojun Liao 已提交
2747
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode,
2748
                                            SExecTaskInfo* pTaskInfo) {
2749 2750 2751 2752 2753 2754
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

2755
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
2756
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2757

2758
  int32_t      numOfCols = 0;
H
Haojun Liao 已提交
2759 2760 2761
  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc);

2762
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
2763 2764 2765 2766
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2767 2768
  initBasicInfo(&pInfo->binfo, pResBlock);

H
Haojun Liao 已提交
2769 2770 2771 2772
  pInfo->twAggSup.waterMark = pSessionNode->window.watermark;
  pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType;
  pInfo->gap = pSessionNode->gap;

2773
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2774 2775
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

2776
  pInfo->tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
L
Liu Jicong 已提交
2777 2778 2779
  pInfo->binfo.pRes = pResBlock;
  pInfo->winSup.prevTs = INT64_MIN;
  pInfo->reptScan = false;
2780
  pInfo->pCondition = pSessionNode->window.node.pConditions;
H
Haojun Liao 已提交
2781

L
Liu Jicong 已提交
2782
  pOperator->name = "SessionWindowAggOperator";
2783
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
2784
  pOperator->blocking = true;
L
Liu Jicong 已提交
2785 2786
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
2787 2788 2789 2790 2791

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL,
                                         destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  pOperator->pTaskInfo = pTaskInfo;
  code = appendDownstream(pOperator, &downstream, 1);
2792 2793 2794 2795
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2796 2797
  return pOperator;

L
Liu Jicong 已提交
2798
_error:
2799
  destroySWindowOperatorInfo(pInfo);
2800 2801 2802
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
2803
}
5
54liuyao 已提交
2804

5
54liuyao 已提交
2805
void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
2806
                      SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) {
5
54liuyao 已提交
2807 2808
  for (int32_t k = 0; k < numOfOutput; ++k) {
    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
2809 2810 2811 2812 2813
      if (!pTimeWindowData) {
        continue;
      }

      SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);
L
Liu Jicong 已提交
2814 2815
      char*                p = GET_ROWCELL_INTERBUF(pEntryInfo);
      SColumnInfoData      idata = {0};
2816 2817 2818 2819 2820 2821 2822 2823
      idata.info.type = TSDB_DATA_TYPE_BIGINT;
      idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
      idata.pData = p;

      SScalarParam out = {.columnData = &idata};
      SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
      pDestCtx[k].sfp.process(&tw, 1, &out);
      pEntryInfo->numOfRes = 1;
L
Liu Jicong 已提交
2824
    } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
2825
      int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
5
54liuyao 已提交
2826 2827 2828
      if (code != TSDB_CODE_SUCCESS) {
        qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
        pTaskInfo->code = code;
2829
        T_LONG_JMP(pTaskInfo->env, code);
5
54liuyao 已提交
2830 2831 2832 2833 2834
      }
    }
  }
}

2835 2836
bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) {
  return TSDB_CODE_SUCCESS == streamStateGet(pState, pKey, NULL, 0);
2837 2838
}

2839 2840 2841 2842 2843
static void rebuildIntervalWindow(SOperatorInfo* pOperator, SExprSupp* pSup, SArray* pWinArray, SHashObj* pUpdatedMap) {
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
  int32_t                      size = taosArrayGetSize(pWinArray);
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
5
54liuyao 已提交
2844 2845 2846
  if (!pInfo->pChildren) {
    return;
  }
5
54liuyao 已提交
2847
  for (int32_t i = 0; i < size; i++) {
H
Haojun Liao 已提交
2848
    SWinKey*    pWinRes = taosArrayGet(pWinArray, i);
2849
    SResultRow* pCurResult = NULL;
2850
    STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval);
2851 2852
    if (isDeletedStreamWindow(&parentWin, pWinRes->groupId, pInfo->pState, &pInfo->twAggSup) &&
        isCloseWindow(&parentWin, &pInfo->twAggSup)) {
2853 2854
      continue;
    }
2855 2856 2857 2858 2859
    int32_t code = setOutputBuf(pInfo->pState, &parentWin, &pCurResult, pWinRes->groupId, pSup->pCtx, numOfOutput,
                                pSup->rowEntryInfoOffset, &pInfo->aggSup);
    if (code != TSDB_CODE_SUCCESS || pCurResult == NULL) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
5
54liuyao 已提交
2860
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
2861
    int32_t num = 0;
5
54liuyao 已提交
2862
    for (int32_t j = 0; j < numOfChildren; j++) {
2863 2864 2865 2866
      SOperatorInfo*               pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SStreamIntervalOperatorInfo* pChInfo = pChildOp->info;
      SExprSupp*                   pChildSup = &pChildOp->exprSupp;
      if (!hasIntervalWindow(pChInfo->pState, pWinRes)) {
2867 2868
        continue;
      }
2869
      num++;
2870
      SResultRow* pChResult = NULL;
2871 2872
      setOutputBuf(pChInfo->pState, &parentWin, &pChResult, pWinRes->groupId, pChildSup->pCtx, pChildSup->numOfExprs,
                   pChildSup->rowEntryInfoOffset, &pChInfo->aggSup);
2873 2874
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true);
      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
2875
    }
2876
    if (num > 0 && pUpdatedMap) {
2877 2878 2879
      saveWinResultInfo(pCurResult->win.skey, pWinRes->groupId, pUpdatedMap);
      saveOutputBuf(pInfo->pState, pWinRes, pCurResult, pInfo->aggSup.resultRowSize);
      releaseOutputBuf(pInfo->pState, pWinRes, pCurResult);
2880
    }
5
54liuyao 已提交
2881 2882 2883 2884 2885
  }
}

bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
  SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId);
2886
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
L
Liu Jicong 已提交
2887
                                                               GET_RES_WINDOW_KEY_LEN(sizeof(int64_t)));
5
54liuyao 已提交
2888 2889 2890
  return p1 == NULL;
}

2891
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) {
5
54liuyao 已提交
2892 2893 2894 2895
  if (pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {
    SWinKey key = {.ts = pWin->skey, .groupId = groupId};
    void*   pVal = NULL;
    int32_t size = 0;
2896
    if (streamStateGet(pState, &key, &pVal, &size) == TSDB_CODE_SUCCESS) {
5
54liuyao 已提交
2897 2898
      return false;
    }
2899
    return true;
5
54liuyao 已提交
2900 2901 2902 2903
  }
  return false;
}

L
Liu Jicong 已提交
2904 2905 2906 2907
int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,
                        STimeWindow* pNextWin) {
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2908 2909 2910 2911
  int32_t prevEndPos = forwardRows - 1 + startPos;
  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
}

H
Haojun Liao 已提交
2912
void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {
5
54liuyao 已提交
2913 2914 2915 2916
  SArray* childIds = taosArrayInit(8, sizeof(int32_t));
  for (int32_t i = 0; i < size; i++) {
    taosArrayPush(childIds, &i);
  }
H
Haojun Liao 已提交
2917
  taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*));
5
54liuyao 已提交
2918 2919 2920 2921
}

static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

2922
static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) {
2923
  tSimpleHashClear(pInfo->aggSup.pResultRowHashTable);
5
54liuyao 已提交
2924
  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
2925
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
L
Liu Jicong 已提交
2926
  pInfo->aggSup.currentPageId = -1;
2927
  streamStateClear(pInfo->pState);
5
54liuyao 已提交
2928 2929
}

5
54liuyao 已提交
2930 2931 2932 2933
static void clearSpecialDataBlock(SSDataBlock* pBlock) {
  if (pBlock->info.rows <= 0) {
    return;
  }
5
54liuyao 已提交
2934 2935 2936
  blockDataCleanup(pBlock);
}

2937
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
5
54liuyao 已提交
2938 2939
  // ASSERT(pDest->info.capacity >= pSource->info.rows);
  blockDataEnsureCapacity(pDest, pSource->info.rows);
5
54liuyao 已提交
2940
  clearSpecialDataBlock(pDest);
5
54liuyao 已提交
2941 2942
  SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
  SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
2943

5
54liuyao 已提交
2944
  // copy timestamp column
2945 2946
  colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info);
  for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) {
5
54liuyao 已提交
2947 2948 2949
    SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
    colDataAppendNNULL(pCol, 0, pSource->info.rows);
  }
2950

5
54liuyao 已提交
2951
  pDest->info.rows = pSource->info.rows;
2952 2953
  pDest->info.groupId = pSource->info.groupId;
  pDest->info.type = pSource->info.type;
5
54liuyao 已提交
2954 2955 2956
  blockDataUpdateTsWindow(pDest, 0);
}

5
54liuyao 已提交
2957 2958 2959 2960 2961 2962
static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
  clearSpecialDataBlock(pBlock);
  int32_t size = taosArrayGetSize(array);
  if (size - (*pIndex) == 0) {
    return;
  }
L
Liu Jicong 已提交
2963
  blockDataEnsureCapacity(pBlock, size - (*pIndex));
5
54liuyao 已提交
2964
  ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
2965 2966 2967 2968 2969
  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
5
54liuyao 已提交
2970
  for (; (*pIndex) < size; (*pIndex)++) {
L
Liu Jicong 已提交
2971
    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));
5
54liuyao 已提交
2972 2973 2974
    colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
    colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
2975 2976
    colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
5
54liuyao 已提交
2977 2978 2979 2980 2981 2982 2983 2984 2985
    pBlock->info.rows++;
  }
  if ((*pIndex) == size) {
    *pIndex = 0;
    taosArrayClear(array);
  }
  blockDataUpdateTsWindow(pBlock, 0);
}

L
Liu Jicong 已提交
2986
void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
5
54liuyao 已提交
2987
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
L
Liu Jicong 已提交
2988
  TSKEY*           tsData = (TSKEY*)pStartCol->pData;
5
54liuyao 已提交
2989
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
L
Liu Jicong 已提交
2990 2991
  uint64_t*        groupIdData = (uint64_t*)pGroupCol->pData;
  int32_t          chId = getChildIndex(pBlock);
5
54liuyao 已提交
2992
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
H
Haojun Liao 已提交
2993 2994
    SWinKey winRes = {.ts = tsData[i], .groupId = groupIdData[i]};
    void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
2995
    if (chIds) {
L
Liu Jicong 已提交
2996
      SArray* chArray = *(SArray**)chIds;
5
54liuyao 已提交
2997 2998
      int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
      if (index != -1) {
5
54liuyao 已提交
2999
        qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId);
5
54liuyao 已提交
3000 3001 3002
        taosArrayRemove(chArray, index);
        if (taosArrayGetSize(chArray) == 0) {
          // pull data is over
5
54liuyao 已提交
3003
          taosArrayDestroy(chArray);
H
Haojun Liao 已提交
3004
          taosHashRemove(pMap, &winRes, sizeof(SWinKey));
5
54liuyao 已提交
3005 3006 3007 3008 3009
        }
      }
    }
  }
}
5
54liuyao 已提交
3010

3011
static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) {
3012 3013
  int32_t size = taosArrayGetSize(wins);
  for (int32_t i = 0; i < size; i++) {
L
Liu Jicong 已提交
3014
    SWinKey*    winKey = taosArrayGet(wins, i);
3015
    STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
3016
    if (needDeleteWindowBuf(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) {
3017 3018 3019 3020 3021
      void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
      if (!chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId};
        // add pull data request
        savePullWindow(&pull, pInfo->pPullWins);
3022 3023 3024
        int32_t size1 = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, winKey, size1);
        qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, size1);
3025 3026 3027 3028 3029
      }
    }
  }
}

5
54liuyao 已提交
3030 3031 3032 3033 3034 3035
static void clearFunctionContext(SExprSupp* pSup) {
  for (int32_t i = 0; i < pSup->numOfExprs; i++) {
    pSup->pCtx[i].saveHandle.currentPage = -1;
  }
}

3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151
void doBuildResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  // set output datablock version
  pBlock->info.version = pTaskInfo->version;

  blockDataCleanup(pBlock);
  if (!hasRemainResults(pGroupResInfo)) {
    return;
  }

  // clear the existed group id
  pBlock->info.groupId = 0;
  buildDataBlockFromGroupRes(pTaskInfo, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
}

static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId,
                                    SHashObj* pUpdatedMap) {
  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;

  SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*  pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*      pSup = &pOperatorInfo->exprSupp;
  int32_t         numOfOutput = pSup->numOfExprs;
  int32_t         step = 1;
  TSKEY*          tsCols = NULL;
  SResultRow*     pResult = NULL;
  int32_t         forwardRows = 0;

  ASSERT(pSDataBlock->pDataBlock != NULL);
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  tsCols = (int64_t*)pColDataInfo->pData;

  int32_t     startPos = 0;
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
  STimeWindow nextWin =
      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
  while (1) {
    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
    if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
      if (startPos < 0) {
        break;
      }
      continue;
    }

    if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) {
      bool    ignore = true;
      SWinKey winRes = {
          .ts = nextWin.skey,
          .groupId = groupId,
      };
      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
      if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && !chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = groupId};
        // add pull data request
        savePullWindow(&pull, pInfo->pPullWins);
        int32_t size = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, &winRes, size);
        qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size);
      } else {
        int32_t index = -1;
        SArray* chArray = NULL;
        int32_t chId = 0;
        if (chIds) {
          chArray = *(void**)chIds;
          chId = getChildIndex(pSDataBlock);
          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
        }
        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {
          ignore = false;
        }
      }

      if (ignore) {
        startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
        if (startPos < 0) {
          break;
        }
        continue;
      }
    }

    int32_t code = setOutputBuf(pInfo->pState, &nextWin, &pResult, groupId, pSup->pCtx, numOfOutput,
                                pSup->rowEntryInfoOffset, &pInfo->aggSup);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
                                           TSDB_ORDER_ASC);
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
      saveWinResultInfo(pResult->win.skey, groupId, pUpdatedMap);
    }
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pSDataBlock->info.rows, numOfOutput);
    SWinKey key = {
        .ts = nextWin.skey,
        .groupId = groupId,
    };
    saveOutputBuf(pInfo->pState, &key, pResult, pInfo->aggSup.resultRowSize);
    releaseOutputBuf(pInfo->pState, &key, pResult);
    if (pInfo->delKey.ts > key.ts) {
      pInfo->delKey = key;
    }
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
    startPos =
        getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);
    if (startPos < 0) {
      break;
    }
  }
}

5
54liuyao 已提交
3152
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
3153
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
3154 3155 3156

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  TSKEY          maxTs = INT64_MIN;
5
54liuyao 已提交
3157
  TSKEY          minTs = INT64_MAX;
5
54liuyao 已提交
3158

3159 3160
  SExprSupp* pSup = &pOperator->exprSupp;

5
54liuyao 已提交
3161
  qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
L
Liu Jicong 已提交
3162

5
54liuyao 已提交
3163 3164 3165
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
3166 3167 3168 3169
    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
    if (pInfo->pPullDataRes->info.rows != 0) {
      // process the rest of the data
      ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3170
      printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3171 3172 3173
      return pInfo->pPullDataRes;
    }

3174 3175 3176 3177 3178 3179 3180
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
      return pInfo->pDelRes;
    }

3181
    doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
3182 3183 3184
    if (pInfo->binfo.pRes->info.rows != 0) {
      printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
      return pInfo->binfo.pRes;
5
54liuyao 已提交
3185
    }
5
54liuyao 已提交
3186 3187 3188 3189 3190 3191 3192 3193

    doSetOperatorCompleted(pOperator);
    if (!IS_FINAL_OP(pInfo)) {
      clearFunctionContext(&pOperator->exprSupp);
      // semi interval operator clear disk buffer
      clearStreamIntervalOperator(pInfo);
      qDebug("===stream===clear semi operator");
    } else {
3194 3195
      deleteIntervalDiscBuf(pInfo->pState, pInfo->pPullDataMap, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark,
                            &pInfo->interval, &pInfo->delKey);
5
54liuyao 已提交
3196 3197
    }
    return NULL;
5
54liuyao 已提交
3198
  } else {
5
54liuyao 已提交
3199
    if (!IS_FINAL_OP(pInfo)) {
3200
      doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
3201
      if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3202
        printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3203 3204
        return pInfo->binfo.pRes;
      }
5
54liuyao 已提交
3205 3206 3207 3208
    }
    if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
      pInfo->returnUpdate = false;
      ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3209
      printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3210 3211
      // process the rest of the data
      return pInfo->pUpdateRes;
5
54liuyao 已提交
3212
    }
3213 3214 3215
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
5
54liuyao 已提交
3216
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
3217 3218
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
3219 3220
  }

5
54liuyao 已提交
3221 3222 3223
  SArray*    pUpdated = taosArrayInit(4, POINTER_BYTES);
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*  pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
5
54liuyao 已提交
3224 3225 3226
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
3227
      clearSpecialDataBlock(pInfo->pUpdateRes);
5
54liuyao 已提交
3228
      removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
5
54liuyao 已提交
3229
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
3230
      qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3231 3232
      break;
    }
5
54liuyao 已提交
3233
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv");
3234

H
Haojun Liao 已提交
3235 3236
    ASSERT(pBlock->info.type != STREAM_INVERT);
    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
5
54liuyao 已提交
3237 3238
      pInfo->binfo.pRes->info.type = pBlock->info.type;
    } else if (pBlock->info.type == STREAM_CLEAR) {
H
Haojun Liao 已提交
3239
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinKey));
3240
      doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins, NULL);
5
54liuyao 已提交
3241
      if (IS_FINAL_OP(pInfo)) {
3242 3243 3244 3245 3246 3247 3248
        int32_t                      childIndex = getChildIndex(pBlock);
        SOperatorInfo*               pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info;
        SExprSupp*                   pChildSup = &pChildOp->exprSupp;

        doDeleteWindows(pChildOp, &pChildInfo->interval, pChildOp->exprSupp.numOfExprs, pBlock, NULL, NULL);
        rebuildIntervalWindow(pOperator, pSup, pUpWins, pUpdatedMap);
5
54liuyao 已提交
3249 3250
        taosArrayDestroy(pUpWins);
        continue;
3251
      }
5
54liuyao 已提交
3252
      removeResults(pUpWins, pUpdatedMap);
3253
      copyDataBlock(pInfo->pUpdateRes, pBlock);
5
54liuyao 已提交
3254
      pInfo->returnUpdate = true;
3255
      taosArrayDestroy(pUpWins);
5
54liuyao 已提交
3256
      break;
3257
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
3258
      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
3259
      doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, delWins, pUpdatedMap);
3260
      if (IS_FINAL_OP(pInfo)) {
3261 3262 3263 3264 3265 3266
        int32_t                      childIndex = getChildIndex(pBlock);
        SOperatorInfo*               pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info;
        SExprSupp*                   pChildSup = &pChildOp->exprSupp;
        doDeleteWindows(pChildOp, &pChildInfo->interval, pChildOp->exprSupp.numOfExprs, pBlock, NULL, NULL);
        rebuildIntervalWindow(pOperator, pSup, delWins, pUpdatedMap);
3267 3268 3269
        addRetriveWindow(delWins, pInfo);
        taosArrayAddAll(pInfo->pDelWins, delWins);
        taosArrayDestroy(delWins);
3270 3271
        continue;
      }
3272 3273 3274
      removeResults(delWins, pUpdatedMap);
      taosArrayAddAll(pInfo->pDelWins, delWins);
      taosArrayDestroy(delWins);
3275
      break;
5
54liuyao 已提交
3276
    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
3277
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
5
54liuyao 已提交
3278
      continue;
5
54liuyao 已提交
3279
    } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
3280
      doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, NULL, pUpdatedMap);
5
54liuyao 已提交
3281 3282 3283 3284
      if (taosArrayGetSize(pUpdated) > 0) {
        break;
      }
      continue;
L
Liu Jicong 已提交
3285 3286
    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) {
      processPullOver(pBlock, pInfo->pPullDataMap);
5
54liuyao 已提交
3287
      continue;
5
54liuyao 已提交
3288
    }
5
54liuyao 已提交
3289

5
54liuyao 已提交
3290 3291 3292 3293
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
3294 3295
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
    doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
5
54liuyao 已提交
3296
    if (IS_FINAL_OP(pInfo)) {
S
shenglian zhou 已提交
3297
      int32_t chIndex = getChildIndex(pBlock);
5
54liuyao 已提交
3298 3299 3300 3301 3302
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
        SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
        if (!pChildOp) {
3303
          T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3304
        }
3305
        SStreamIntervalOperatorInfo* pTmpInfo = pChildOp->info;
3306
        pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3307
        taosArrayPush(pInfo->pChildren, &pChildOp);
5
54liuyao 已提交
3308
        qDebug("===stream===add child, id:%d", chIndex);
5
54liuyao 已提交
3309
      }
3310 3311 3312 3313
      SOperatorInfo*               pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
      SStreamIntervalOperatorInfo* pChInfo = pChildOp->info;
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
      doStreamIntervalAggImpl(pChildOp, pBlock, pBlock->info.groupId, NULL);
5
54liuyao 已提交
3314
    }
5
54liuyao 已提交
3315 3316 3317
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
    maxTs = TMAX(maxTs, pBlock->info.watermark);
    minTs = TMIN(minTs, pBlock->info.window.skey);
5
54liuyao 已提交
3318
  }
S
shenglian zhou 已提交
3319

5
54liuyao 已提交
3320
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
3321
  pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs);
5
54liuyao 已提交
3322
  if (IS_FINAL_OP(pInfo)) {
3323 3324 3325
    closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval,
                              pInfo->pPullDataMap, pUpdatedMap, pOperator);
    closeChildIntervalWindow(pOperator, pInfo->pChildren, pInfo->twAggSup.maxTs);
5
54liuyao 已提交
3326
  }
3327
  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;
5
54liuyao 已提交
3328

5
54liuyao 已提交
3329 3330 3331 3332 3333 3334 3335
  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosHashCleanup(pUpdatedMap);
  taosArraySort(pUpdated, resultrowComparAsc);

5
54liuyao 已提交
3336 3337
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
3338 3339 3340 3341 3342

  doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
  if (pInfo->pPullDataRes->info.rows != 0) {
    // process the rest of the data
    ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3343
    printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3344 3345 3346
    return pInfo->pPullDataRes;
  }

5
54liuyao 已提交
3347 3348 3349 3350 3351 3352 3353
  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows != 0) {
    // process the rest of the data
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
    return pInfo->pDelRes;
  }

3354
  doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
3355
  if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3356
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3357 3358 3359 3360 3361 3362
    return pInfo->binfo.pRes;
  }

  if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
    pInfo->returnUpdate = false;
    ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3363
    printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3364 3365 3366
    // process the rest of the data
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
3367 3368 3369
  return NULL;
}

S
shenglian zhou 已提交
3370 3371
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                     SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
3372 3373 3374
  SIntervalPhysiNode*          pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3375 3376 3377
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
3378

3379
  pOperator->pTaskInfo = pTaskInfo;
S
shenglian zhou 已提交
3380 3381 3382 3383 3384 3385 3386 3387
  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                .sliding = pIntervalPhyNode->sliding,
                                .intervalUnit = pIntervalPhyNode->intervalUnit,
                                .slidingUnit = pIntervalPhyNode->slidingUnit,
                                .offset = pIntervalPhyNode->offset,
                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pIntervalPhyNode->window.watermark,
5
54liuyao 已提交
3388 3389
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
3390
      .minTs = INT64_MAX,
3391
      .deleteMark = INT64_MAX,
S
shenglian zhou 已提交
3392
  };
3393
  ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY);
5
54liuyao 已提交
3394 3395
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
3396
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3397 3398 3399 3400 3401 3402 3403 3404 3405
  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

S
shenglian zhou 已提交
3406 3407
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5
54liuyao 已提交
3408
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
3409 3410

  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
3411 3412 3413 3414
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

H
Haojun Liao 已提交
3415
  initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs);
3416 3417
  initBasicInfo(&pInfo->binfo, pResBlock);

3418
  ASSERT(numOfCols > 0);
5
54liuyao 已提交
3419
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
3420

3421 3422 3423 3424
  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
  *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
  streamStateSetNumber(pInfo->pState, -1);

3425
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3426 3427
  pInfo->pChildren = NULL;
  if (numOfChild > 0) {
3428
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
5
54liuyao 已提交
3429 3430 3431
    for (int32_t i = 0; i < numOfChild; i++) {
      SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp) {
3432
        SStreamIntervalOperatorInfo* pChInfo = pChildOp->info;
3433
        pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3434
        taosArrayPush(pInfo->pChildren, &pChildOp);
3435
        streamStateSetNumber(pChInfo->pState, i);
5
54liuyao 已提交
3436 3437 3438 3439 3440
        continue;
      }
      goto _error;
    }
  }
3441
  pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
3442
  blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
5
54liuyao 已提交
3443 3444
  pInfo->returnUpdate = false;

3445
  pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
5
54liuyao 已提交
3446

5
54liuyao 已提交
3447 3448 3449 3450
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
    pInfo->isFinal = true;
    pOperator->name = "StreamFinalIntervalOperator";
  } else {
5
54liuyao 已提交
3451
    // semi interval operator does not catch result
5
54liuyao 已提交
3452 3453
    pInfo->isFinal = false;
    pOperator->name = "StreamSemiIntervalOperator";
H
Haojun Liao 已提交
3454
    ASSERT(pInfo->aggSup.currentPageId == -1);
5
54liuyao 已提交
3455 3456
  }

5
54liuyao 已提交
3457
  if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
5
54liuyao 已提交
3458 3459
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }
5
54liuyao 已提交
3460 3461 3462 3463
  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
  pInfo->pullIndex = 0;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
3464
  pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
5
54liuyao 已提交
3465
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
3466
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3467
  pInfo->delIndex = 0;
H
Haojun Liao 已提交
3468
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
3469 3470
  pInfo->delKey.ts = INT64_MAX;
  pInfo->delKey.groupId = 0;
5
54liuyao 已提交
3471

5
54liuyao 已提交
3472
  pOperator->operatorType = pPhyNode->type;
5
54liuyao 已提交
3473 3474 3475 3476
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;

S
shenglian zhou 已提交
3477 3478 3479
  pOperator->fpSet =
      createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
3480
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
5
54liuyao 已提交
3481
    initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
3482
  }
5
54liuyao 已提交
3483 3484 3485 3486 3487 3488 3489 3490
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
3491
  destroyStreamFinalIntervalOperatorInfo(pInfo);
5
54liuyao 已提交
3492 3493 3494
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
5
54liuyao 已提交
3495
}
5
54liuyao 已提交
3496 3497 3498

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
3499
  void** pIte = NULL;
3500
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
3501
    SArray* pWins = (SArray*)(*pIte);
3502 3503 3504
    taosArrayDestroy(pWins);
  }
  taosHashCleanup(pSup->pResultRows);
5
54liuyao 已提交
3505
  destroyDiskbasedBuf(pSup->pResultBuf);
3506
  blockDataDestroy(pSup->pScanBlock);
5
54liuyao 已提交
3507 3508
}

5
54liuyao 已提交
3509 3510 3511 3512
void destroyStateWinInfo(void* ptr) {
  if (ptr == NULL) {
    return;
  }
3513
  SStateWindowInfo* pWin = (SStateWindowInfo*)ptr;
5
54liuyao 已提交
3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528
  taosMemoryFreeClear(pWin->stateKey.pData);
}

void destroyStateStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
  void** pIte = NULL;
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
    SArray* pWins = (SArray*)(*pIte);
    taosArrayDestroyEx(pWins, (FDelete)destroyStateWinInfo);
  }
  taosHashCleanup(pSup->pResultRows);
  destroyDiskbasedBuf(pSup->pResultBuf);
  blockDataDestroy(pSup->pScanBlock);
}

3529
void destroyStreamSessionAggOperatorInfo(void* param) {
5
54liuyao 已提交
3530
  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;
3531
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
3532 3533
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
3534 3535 3536
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
3537
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
3538
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
3539
      destroyStreamSessionAggOperatorInfo(pChInfo);
3540 3541 3542
      taosMemoryFreeClear(pChild);
    }
  }
5
54liuyao 已提交
3543 3544 3545 3546
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  blockDataDestroy(pInfo->pWinBlock);
  blockDataDestroy(pInfo->pUpdateRes);
5
54liuyao 已提交
3547 3548
  destroySqlFunctionCtx(pInfo->pDummyCtx, 0);
  taosHashCleanup(pInfo->pStDeleted);
3549

D
dapan1121 已提交
3550
  taosMemoryFreeClear(param);
5
54liuyao 已提交
3551 3552
}

3553 3554
int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,
                        SSDataBlock* pResultBlock) {
3555 3556 3557 3558
  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
  if (code != TSDB_CODE_SUCCESS) {
    return code;
  }
H
Haojun Liao 已提交
3559
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
3560

3561
  initBasicInfo(pBasicInfo, pResultBlock);
3562

5
54liuyao 已提交
3563
  for (int32_t i = 0; i < numOfCols; ++i) {
3564
    pSup->pCtx[i].saveHandle.pBuf = NULL;
5
54liuyao 已提交
3565
  }
3566

3567
  ASSERT(numOfCols > 0);
5
54liuyao 已提交
3568 3569 3570 3571 3572 3573 3574 3575
  return TSDB_CODE_SUCCESS;
}

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {
  for (int i = 0; i < nums; i++) {
    pDummy[i].functionId = pCtx[i].functionId;
  }
}
5
54liuyao 已提交
3576

X
Xiaoyu Wang 已提交
3577
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
3578 3579 3580 3581 3582 3583 3584 3585 3586 3587
                    uint16_t type, int32_t tsColIndex) {
  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
    pScanInfo->tsColIndex = tsColIndex;
  }

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
    initDownStream(downstream->pDownstream[0], pAggSup, gap, waterMark, type, tsColIndex);
    return;
  }
3588
  SStreamScanInfo* pScanInfo = downstream->info;
3589
  pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
5
54liuyao 已提交
3590
  pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
5
54liuyao 已提交
3591 3592
}

3593 3594
int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx,
                                int32_t numOfOutput) {
3595 3596 3597
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo));
}

3598 3599 3600 3601 3602 3603 3604
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                  SExecTaskInfo* pTaskInfo) {
  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
  int32_t                        numOfCols = 0;
  SExprInfo*                     pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock*                   pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;
X
Xiaoyu Wang 已提交
3605
  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
3606
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3607 3608 3609 3610
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

3611 3612
  pOperator->pTaskInfo = pTaskInfo;

3613
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3614 3615 3616
  if (pSessionNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar);
3617
    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
5
54liuyao 已提交
3618 3619 3620 3621
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
3622
  SExprSupp* pSup = &pOperator->exprSupp;
5
54liuyao 已提交
3623

3624
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
3625 3626 3627
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
3628

3629
  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
3630 3631 3632
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
X
Xiaoyu Wang 已提交
3633

5
54liuyao 已提交
3634 3635 3636 3637
  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }
3638
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
3639

3640
  pInfo->twAggSup = (STimeWindowAggSupp){
5
54liuyao 已提交
3641 3642 3643 3644 3645
      .waterMark = pSessionNode->window.watermark,
      .calTrigger = pSessionNode->window.triggerType,
      .maxTs = INT64_MIN,
      .minTs = INT64_MAX,
  };
H
Haojun Liao 已提交
3646 3647

  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3648 3649
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

5
54liuyao 已提交
3650 3651 3652 3653
  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  if (pSessionNode->window.pTsEnd) {
    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;
  }
3654
  pInfo->gap = pSessionNode->gap;
5
54liuyao 已提交
3655 3656 3657 3658 3659
  pInfo->binfo.pRes = pResBlock;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
5
54liuyao 已提交
3660
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3661
  pInfo->pChildren = NULL;
5
54liuyao 已提交
3662 3663
  pInfo->isFinal = false;
  pInfo->pPhyNode = pPhyNode;
5
54liuyao 已提交
3664
  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
5
54liuyao 已提交
3665 3666

  pOperator->name = "StreamSessionWindowAggOperator";
3667
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
5
54liuyao 已提交
3668 3669 3670
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
3671 3672 3673
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
3674
  if (downstream) {
L
Liu Jicong 已提交
3675 3676
    initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType,
                   pInfo->primaryTsIndex);
5
54liuyao 已提交
3677 3678
    code = appendDownstream(pOperator, &downstream, 1);
  }
5
54liuyao 已提交
3679 3680 3681 3682
  return pOperator;

_error:
  if (pInfo != NULL) {
3683
    destroyStreamSessionAggOperatorInfo(pInfo);
5
54liuyao 已提交
3684 3685 3686 3687 3688 3689 3690 3691
  }

  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

int64_t getSessionWindowEndkey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
3692
  SArray*            pWinInfos = (SArray*)data;
5
54liuyao 已提交
3693 3694 3695
  SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index);
  return pWin->win.ekey;
}
5
54liuyao 已提交
3696 3697

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {
5
54liuyao 已提交
3698
  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {
5
54liuyao 已提交
3699 3700 3701 3702 3703
    return true;
  }
  return false;
}

3704
bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); }
5
54liuyao 已提交
3705

3706
static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs, int32_t index) {
L
Liu Jicong 已提交
3707 3708
  SResultWindowInfo win = {
      .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
5
54liuyao 已提交
3709 3710 3711
  return taosArrayInsert(pWinInfos, index, &win);
}

3712
static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs) {
L
Liu Jicong 已提交
3713 3714
  SResultWindowInfo win = {
      .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
5
54liuyao 已提交
3715 3716 3717
  return taosArrayPush(pWinInfos, &win);
}

3718
SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) {
3719
  void**  ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t));
3720 3721 3722
  SArray* pWinInfos = NULL;
  if (ite == NULL) {
    pWinInfos = taosArrayInit(1024, pAggSup->valueSize);
3723
    taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void*));
3724 3725 3726 3727 3728 3729
  } else {
    pWinInfos = *ite;
  }
  return pWinInfos;
}

5
54liuyao 已提交
3730 3731
// don't add new window
SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
3732
                                       int64_t gap, int32_t* pIndex) {
3733 3734
  STimeWindow searchWin = {.skey = startTs, .ekey = endTs};
  SArray*     pWinInfos = getWinInfos(pAggSup, groupId);
5
54liuyao 已提交
3735 3736 3737 3738 3739 3740 3741
  pAggSup->pCurWins = pWinInfos;

  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    return NULL;
  }
  // find the first position which is smaller than the key
3742
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3743 3744 3745
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
3746
    if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) {
5
54liuyao 已提交
3747 3748 3749 3750 3751 3752 3753
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
3754
    if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) {
5
54liuyao 已提交
3755 3756 3757
      *pIndex = index + 1;
      return pWin;
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
3758
      *pIndex = index + 1;
5
54liuyao 已提交
3759 3760 3761 3762 3763 3764 3765
      return pWin;
    }
  }

  return NULL;
}

3766 3767
SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
                                        int64_t gap, int32_t* pIndex) {
3768 3769 3770
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

5
54liuyao 已提交
3771 3772
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
5
54liuyao 已提交
3773
    *pIndex = 0;
3774
    return addNewSessionWindow(pWinInfos, startTs, endTs);
5
54liuyao 已提交
3775 3776
  }
  // find the first position which is smaller than the key
5
54liuyao 已提交
3777
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3778 3779 3780
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
5
54liuyao 已提交
3781
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3782 3783 3784 3785 3786 3787 3788
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
5
54liuyao 已提交
3789
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3790 3791
      *pIndex = index + 1;
      return pWin;
5
54liuyao 已提交
3792 3793 3794
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
      *pIndex = index;
      return pWin;
5
54liuyao 已提交
3795 3796 3797 3798 3799
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
3800
    return addNewSessionWindow(pWinInfos, startTs, endTs);
5
54liuyao 已提交
3801
  }
5
54liuyao 已提交
3802
  *pIndex = index + 1;
3803
  return insertNewSessionWindow(pWinInfos, startTs, endTs, index + 1);
5
54liuyao 已提交
3804 3805
}

dengyihao's avatar
dengyihao 已提交
3806 3807
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId,
                                int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted) {
5
54liuyao 已提交
3808
  for (int32_t i = start; i < rows; ++i) {
3809
    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {
5
54liuyao 已提交
3810 3811
      return i - start;
    }
5
54liuyao 已提交
3812
    if (pWinInfo->win.skey > pStartTs[i]) {
5
54liuyao 已提交
3813
      if (pStDeleted && pWinInfo->isOutput) {
H
Haojun Liao 已提交
3814
        SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
3815
        taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
3816 3817
        pWinInfo->isOutput = false;
      }
5
54liuyao 已提交
3818 3819 3820 3821 3822
      pWinInfo->win.skey = pStartTs[i];
    }
    pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pStartTs[i]);
    if (pEndTs) {
      pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pEndTs[i]);
5
54liuyao 已提交
3823 3824 3825 3826 3827
    }
  }
  return rows - start;
}

X
Xiaoyu Wang 已提交
3828
static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,
3829
                                  uint64_t groupId, int32_t numOfOutput, int32_t* rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3830
                                  SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) {
5
54liuyao 已提交
3831 3832
  assert(pWinInfo->win.skey <= pWinInfo->win.ekey);
  // too many time window in query
3833
  int32_t size = taosArrayGetSize(pAggSup->pCurWins);
3834
  if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH && size > MAX_INTERVAL_TIME_WINDOW) {
3835
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
5
54liuyao 已提交
3836
  }
X
Xiaoyu Wang 已提交
3837

5
54liuyao 已提交
3838
  if (pWinInfo->pos.pageId == -1) {
3839
    *pResult = getNewResultRow(pAggSup->pResultBuf, &pAggSup->currentPageId, pAggSup->resultRowSize);
5
54liuyao 已提交
3840 3841 3842 3843 3844 3845 3846 3847
    if (*pResult == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }

    // add a new result set for a new group
    pWinInfo->pos.pageId = (*pResult)->pageId;
    pWinInfo->pos.offset = (*pResult)->offset;
  } else {
3848
    *pResult = getResultRowByPos(pAggSup->pResultBuf, &pWinInfo->pos, true);
5
54liuyao 已提交
3849 3850 3851 3852 3853 3854 3855 3856
    if (!(*pResult)) {
      qError("getResultRowByPos return NULL, TID:%s", GET_TASKID(pTaskInfo));
      return TSDB_CODE_FAILED;
    }
  }

  // set time window for current result
  (*pResult)->win = pWinInfo->win;
3857
  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
5
54liuyao 已提交
3858 3859 3860
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3861 3862 3863
static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup,
                                  SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock,
                                  SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3864
                                  int32_t numOutput, SOperatorInfo* pOperator) {
3865
  SExprSupp*     pSup = &pOperator->exprSupp;
3866 3867
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

X
Xiaoyu Wang 已提交
3868 3869
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId);
  TSKEY*           tsCols = (int64_t*)pColDataInfo->pData;
3870 3871
  int32_t          code = setWindowOutputBuf(pCurWin, pResult, pSup->pCtx, pSDataBlock->info.groupId, numOutput,
                                             pSup->rowEntryInfoOffset, pAggSup, pTaskInfo);
5
54liuyao 已提交
3872 3873 3874
  if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }
5
54liuyao 已提交
3875
  updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false);
3876
  doApplyFunctions(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, pSDataBlock->info.rows, numOutput);
5
54liuyao 已提交
3877 3878 3879
  SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pCurWin->pos.pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pAggSup->pResultBuf, bufPage);
5
54liuyao 已提交
3880 3881 3882
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3883 3884
static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                              SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3885
                              int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3886
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3887
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3888 3889
}

X
Xiaoyu Wang 已提交
3890 3891
static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                                   SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex,
3892
                                   int32_t winRows, int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3893
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3894
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3895 3896
}

5
54liuyao 已提交
3897 3898
int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) {
  SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex);
X
Xiaoyu Wang 已提交
3899
  int32_t            size = taosArrayGetSize(pWinInfos);
5
54liuyao 已提交
3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i);
    if (!isInWindow(pCurWin, pWinInfo->win.skey, gap)) {
      return i - startIndex - 1;
    }
  }

  return size - startIndex - 1;
}

5
54liuyao 已提交
3911
void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId,
3912
                       int32_t numOfOutput, SHashObj* pStUpdated, SHashObj* pStDeleted, SOperatorInfo* pOperator) {
3913
  SExprSupp*     pSup = &pOperator->exprSupp;
3914 3915
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

3916
  SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex);
X
Xiaoyu Wang 已提交
3917
  SResultRow*        pCurResult = NULL;
3918
  setWindowOutputBuf(pCurWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3919
                     &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3920
  num += startIndex + 1;
3921
  ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins));
5
54liuyao 已提交
3922 3923
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < num; i++) {
3924
    SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i);
X
Xiaoyu Wang 已提交
3925
    SResultRow*        pWinResult = NULL;
3926
    setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3927
                       &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3928
    pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey);
3929 3930
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true);
    compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
5
54liuyao 已提交
3931
    taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition));
3932
    if (pWinInfo->isOutput && pStDeleted) {
H
Haojun Liao 已提交
3933
      SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
3934
      taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
3935 3936
      pWinInfo->isOutput = false;
    }
3937
    taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
5
54liuyao 已提交
3938 3939
    SFilePage* tmpPage = getBufPage(pInfo->streamAggSup.pResultBuf, pWinInfo->pos.pageId);
    releaseBufPage(pInfo->streamAggSup.pResultBuf, tmpPage);
5
54liuyao 已提交
3940
  }
5
54liuyao 已提交
3941 3942 3943 3944
  SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pCurWin->pos.pageId);
  ASSERT(num > 0);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
5
54liuyao 已提交
3945 3946
}

3947 3948
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
                                   SHashObj* pStDeleted, bool hasEndTs) {
X
Xiaoyu Wang 已提交
3949
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3950
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3951
  bool                           masterScan = true;
3952
  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
5
54liuyao 已提交
3953
  uint64_t                       groupId = pSDataBlock->info.groupId;
X
Xiaoyu Wang 已提交
3954 3955 3956 3957 3958
  int64_t                        gap = pInfo->gap;
  int64_t                        code = TSDB_CODE_SUCCESS;

  int32_t     step = 1;
  bool        ascScan = true;
5
54liuyao 已提交
3959 3960
  TSKEY*      startTsCols = NULL;
  TSKEY*      endTsCols = NULL;
5
54liuyao 已提交
3961
  SResultRow* pResult = NULL;
X
Xiaoyu Wang 已提交
3962
  int32_t     winRows = 0;
5
54liuyao 已提交
3963

5
54liuyao 已提交
3964 3965 3966 3967 3968 3969
  ASSERT(pSDataBlock->pDataBlock);
  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  startTsCols = (int64_t*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = NULL;
  if (hasEndTs) {
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);
5
54liuyao 已提交
3970
  } else {
5
54liuyao 已提交
3971
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
3972
  }
5
54liuyao 已提交
3973
  endTsCols = (int64_t*)pEndTsCol->pData;
X
Xiaoyu Wang 已提交
3974

5
54liuyao 已提交
3975
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
3976
  for (int32_t i = 0; i < pSDataBlock->info.rows;) {
5
54liuyao 已提交
3977
    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
3978 3979 3980
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
3981
    int32_t            winIndex = 0;
3982
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex);
dengyihao's avatar
dengyihao 已提交
3983 3984
    winRows = updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, groupId, pSDataBlock->info.rows, i, pInfo->gap,
                                      pStDeleted);
3985
    code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
3986
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
3987
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
3988
    }
5
54liuyao 已提交
3989

3990
    int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap);
5
54liuyao 已提交
3991
    if (winNum > 0) {
3992
      compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, pStDeleted, pOperator);
5
54liuyao 已提交
3993
    }
5
54liuyao 已提交
3994
    pCurWin->isClosed = false;
5
54liuyao 已提交
3995
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
H
Haojun Liao 已提交
3996 3997
      SWinKey value = {.ts = pCurWin->win.skey, .groupId = groupId};
      code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
5
54liuyao 已提交
3998
      if (code != TSDB_CODE_SUCCESS) {
3999
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4000 4001
      }
      pCurWin->isOutput = true;
5
54liuyao 已提交
4002 4003 4004 4005 4006
    }
    i += winRows;
  }
}

5
54liuyao 已提交
4007
void deleteWindow(SArray* pWinInfos, int32_t index, FDelete fp) {
5
54liuyao 已提交
4008
  ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos));
5
54liuyao 已提交
4009 4010 4011 4012
  if (fp) {
    void* ptr = taosArrayGet(pWinInfos, index);
    fp(ptr);
  }
5
54liuyao 已提交
4013 4014 4015
  taosArrayRemove(pWinInfos, index);
}

4016 4017
static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result,
                                FDelete fp) {
5
54liuyao 已提交
4018
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
4019
  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;
5
54liuyao 已提交
4020
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
4021
  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;
4022
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
4023
  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;
5
54liuyao 已提交
4024
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
4025 4026 4027 4028 4029 4030 4031
    int32_t            winIndex = 0;
    SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex);
    if (!pCurWin) {
      continue;
    }

    do {
4032
      SResultWindowInfo delWin = *pCurWin;
5
54liuyao 已提交
4033
      deleteWindow(pAggSup->pCurWins, winIndex, fp);
5
54liuyao 已提交
4034
      if (result) {
4035 4036
        delWin.groupId = gpDatas[i];
        taosArrayPush(result, &delWin);
5
54liuyao 已提交
4037
      }
4038 4039 4040 4041 4042
      if (winIndex >= taosArrayGetSize(pAggSup->pCurWins)) {
        break;
      }
      pCurWin = taosArrayGet(pAggSup->pCurWins, winIndex);
    } while (pCurWin->win.skey <= endDatas[i]);
5
54liuyao 已提交
4043 4044 4045
  }
}

4046 4047
static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock, int32_t tsIndex,
                                  int32_t numOfOutput, int64_t gap, SArray* result) {
5
54liuyao 已提交
4048
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
4049
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
H
Haojun Liao 已提交
4050 4051
  SColumnInfoData* pGpDataInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*        gpCols = (uint64_t*)pGpDataInfo->pData;
X
Xiaoyu Wang 已提交
4052
  int32_t          step = 0;
5
54liuyao 已提交
4053
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
4054
    int32_t            winIndex = 0;
H
Haojun Liao 已提交
4055
    SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, tsCols[i], INT64_MIN, gpCols[i], gap, &winIndex);
5
54liuyao 已提交
4056
    if (!pCurWin || pCurWin->pos.pageId == -1) {
5
54liuyao 已提交
4057
      // window has been closed.
5
54liuyao 已提交
4058
      step = 1;
5
54liuyao 已提交
4059 4060
      continue;
    }
5
54liuyao 已提交
4061
    step = updateSessionWindowInfo(pCurWin, tsCols, NULL, 0, pBlock->info.rows, i, gap, NULL);
5
54liuyao 已提交
4062
    ASSERT(isInWindow(pCurWin, tsCols[i], gap));
4063
    doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
4064
    if (result) {
4065
      pCurWin->groupId = gpCols[i];
4066 4067
      taosArrayPush(result, pCurWin);
    }
5
54liuyao 已提交
4068 4069 4070
  }
}

5
54liuyao 已提交
4071
static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) {
X
Xiaoyu Wang 已提交
4072
  void*  pData = NULL;
5
54liuyao 已提交
4073
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
4074
  while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) {
5
54liuyao 已提交
4075 4076 4077 4078 4079 4080
    void* key = taosHashGetKey(pData, &keyLen);
    ASSERT(keyLen == sizeof(SResultRowPosition));
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    if (pos == NULL) {
      return TSDB_CODE_QRY_OUT_OF_MEMORY;
    }
H
Haojun Liao 已提交
4081
    pos->groupId = ((SWinKey*)pData)->groupId;
5
54liuyao 已提交
4082
    pos->pos = *(SResultRowPosition*)key;
H
Haojun Liao 已提交
4083
    *(int64_t*)pos->key = ((SWinKey*)pData)->ts;
5
54liuyao 已提交
4084 4085
    taosArrayPush(pUpdated, &pos);
  }
5
54liuyao 已提交
4086
  taosArraySort(pUpdated, resultrowComparAsc);
5
54liuyao 已提交
4087 4088 4089 4090 4091
  return TSDB_CODE_SUCCESS;
}

void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
  blockDataCleanup(pBlock);
4092 4093 4094 4095 4096
  int32_t size = taosHashGetSize(pStDeleted);
  if (size == 0) {
    return;
  }
  blockDataEnsureCapacity(pBlock, size);
5
54liuyao 已提交
4097
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
4098
  while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
H
Haojun Liao 已提交
4099
    SWinKey*         res = *Ite;
4100 4101 4102 4103 4104 4105
    SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
    colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->ts, false);
    SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
    colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->ts, false);
    SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
    colDataAppendNULL(pUidCol, pBlock->info.rows);
5
54liuyao 已提交
4106 4107
    SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
    colDataAppend(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false);
4108 4109 4110 4111
    SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
    colDataAppendNULL(pCalStCol, pBlock->info.rows);
    SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
    colDataAppendNULL(pCalEdCol, pBlock->info.rows);
5
54liuyao 已提交
4112 4113 4114 4115 4116 4117 4118 4119 4120 4121
    pBlock->info.rows += 1;
    if (pBlock->info.rows + 1 >= pBlock->info.capacity) {
      break;
    }
  }
  if ((*Ite) == NULL) {
    taosHashClear(pStDeleted);
  }
}

L
Liu Jicong 已提交
4122
static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t numOfOutput,
4123
                              SOperatorInfo* pOperator, SHashObj* pStUpdated) {
4124
  SExprSupp*     pSup = &pOperator->exprSupp;
4125
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
4126
  int32_t        size = taosArrayGetSize(pWinArray);
4127
  ASSERT(pInfo->pChildren);
4128

4129 4130
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i);
4131
    uint64_t           groupId = pParentWin->groupId;
4132
    int32_t            numOfChildren = taosArrayGetSize(pInfo->pChildren);
4133
    for (int32_t j = 0; j < numOfChildren; j++) {
X
Xiaoyu Wang 已提交
4134
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
4135
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
4136
      SArray*                        pChWins = getWinInfos(&pChInfo->streamAggSup, groupId);
X
Xiaoyu Wang 已提交
4137 4138
      int32_t                        chWinSize = taosArrayGetSize(pChWins);
      int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
4139
      if (index < 0) {
4140
        index = 0;
5
54liuyao 已提交
4141 4142
      }
      for (int32_t k = index; k < chWinSize; k++) {
5
54liuyao 已提交
4143 4144
        SResultWindowInfo* pChWin = taosArrayGet(pChWins, k);
        if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) {
4145 4146 4147 4148 4149 4150
          int32_t            winIndex = 0;
          SResultWindowInfo* pNewParWin =
              getSessionTimeWindow(&pInfo->streamAggSup, pChWin->win.skey, pChWin->win.ekey, groupId, 0, &winIndex);
          SResultRow* pPareResult = NULL;
          setWindowOutputBuf(pNewParWin, &pPareResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
                             &pInfo->streamAggSup, pTaskInfo);
4151
          SResultRow* pChResult = NULL;
5
54liuyao 已提交
4152
          setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
4153
                             pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
4154
          updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pNewParWin->win, true);
4155
          compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
4156 4157 4158 4159 4160 4161

          int32_t winNum = getNumCompactWindow(pInfo->streamAggSup.pCurWins, winIndex, pInfo->gap);
          if (winNum > 0) {
            compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, NULL, pOperator);
          }

5
54liuyao 已提交
4162 4163
          SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
          releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage);
4164 4165 4166 4167 4168 4169

          bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pNewParWin->pos.pageId);
          setBufPageDirty(bufPage, true);
          releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
          SWinKey value = {.ts = pNewParWin->win.skey, .groupId = groupId};
          taosHashPut(pStUpdated, &pNewParWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
5
54liuyao 已提交
4170 4171
        } else if (!pChWin->isClosed) {
          break;
4172 4173 4174 4175 4176 4177
        }
      }
    }
  }
}

X
Xiaoyu Wang 已提交
4178
typedef SResultWindowInfo* (*__get_win_info_)(void*);
5
54liuyao 已提交
4179 4180
SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
5
54liuyao 已提交
4181

4182
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn,
5
54liuyao 已提交
4183
                           bool delete, FDelete fp) {
5
54liuyao 已提交
4184
  // Todo(liuyao) save window to tdb
4185
  void** pIte = NULL;
5
54liuyao 已提交
4186
  size_t keyLen = 0;
4187
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
5
54liuyao 已提交
4188
    uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen);
4189 4190
    SArray*   pWins = (SArray*)(*pIte);
    int32_t   size = taosArrayGetSize(pWins);
4191 4192 4193
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
5
54liuyao 已提交
4194
      if (isCloseWindow(&pSeWin->win, pTwSup)) {
4195 4196
        if (!pSeWin->isClosed) {
          pSeWin->isClosed = true;
5
54liuyao 已提交
4197
          if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {
5
54liuyao 已提交
4198
            int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed);
5
54liuyao 已提交
4199 4200 4201
            if (code != TSDB_CODE_SUCCESS) {
              return code;
            }
4202 4203
            pSeWin->isOutput = true;
          }
5
54liuyao 已提交
4204
          if (delete) {
5
54liuyao 已提交
4205
            deleteWindow(pWins, i, fp);
5
54liuyao 已提交
4206 4207 4208
            i--;
            size = taosArrayGetSize(pWins);
          }
5
54liuyao 已提交
4209
        }
4210
        continue;
5
54liuyao 已提交
4211
      }
4212
      break;
5
54liuyao 已提交
4213 4214 4215 4216 4217
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
4218
static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs, bool delete, FDelete fp) {
5
54liuyao 已提交
4219 4220 4221 4222 4223
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);
    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4224
    closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL, getResWinForSession, delete, fp);
5
54liuyao 已提交
4225 4226 4227
  }
}

4228
int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) {
4229
  void** pIte = NULL;
4230
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
4231
    SArray* pWins = (SArray*)(*pIte);
4232 4233 4234 4235 4236 4237 4238 4239
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (!pSeWin->isClosed) {
        int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed);
        pSeWin->isOutput = true;
      }
5
54liuyao 已提交
4240 4241 4242 4243 4244
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
4245 4246 4247 4248
static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) {
  int32_t size = taosArrayGetSize(pResWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i);
H
Haojun Liao 已提交
4249
    SWinKey            res = {.ts = pWinInfo->win.skey, .groupId = pWinInfo->groupId};
4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262
    taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
  }
}

static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) {
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWin = taosArrayGet(pWins, i);
    taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition));
  }
}

int32_t compareWinKey(void* pKey, void* data, int32_t index) {
L
Liu Jicong 已提交
4263
  SArray*     res = (SArray*)data;
4264
  SResKeyPos* pos = taosArrayGetP(res, index);
L
Liu Jicong 已提交
4265
  SWinKey*    pData = (SWinKey*)pKey;
4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287
  if (pData->ts == *(int64_t*)pos->key) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->ts > *(int64_t*)pos->key) {
    return 1;
  }
  return -1;
}

static void removeSessionDeleteResults(SArray* update, SHashObj* pStDeleted) {
  int32_t size = taosHashGetSize(pStDeleted);
  if (size == 0) {
    return;
  }

  int32_t num = taosArrayGetSize(update);
  for (int32_t i = 0; i < num; i++) {
    SResKeyPos* pos = taosArrayGetP(update, i);
L
Liu Jicong 已提交
4288
    SWinKey     winKey = {.ts = *(int64_t*)pos->key, .groupId = pos->groupId};
4289
    taosHashRemove(pStDeleted, &winKey, sizeof(SWinKey));
5
54liuyao 已提交
4290 4291 4292
  }
}

5
54liuyao 已提交
4293
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
5
54liuyao 已提交
4294
  SExprSupp*                     pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4295
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4296
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
5
54liuyao 已提交
4297 4298 4299 4300
  TSKEY                          maxTs = INT64_MIN;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
4301 4302
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4303
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4304 4305
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
4306
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
4307
    if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
5
54liuyao 已提交
4308 4309
      doSetOperatorCompleted(pOperator);
    }
5
54liuyao 已提交
4310
    printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4311 4312 4313
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
4314 4315
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
4316
  SOperatorInfo* downstream = pOperator->pDownstream[0];
L
Liu Jicong 已提交
4317
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);  // SResKeyPos
5
54liuyao 已提交
4318 4319 4320 4321 4322
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4323
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv");
4324

5
54liuyao 已提交
4325
    if (pBlock->info.type == STREAM_CLEAR) {
X
Xiaoyu Wang 已提交
4326
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
H
Haojun Liao 已提交
4327 4328
      doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, START_TS_COLUMN_INDEX,
                            pOperator->exprSupp.numOfExprs, 0, pWins);
5
54liuyao 已提交
4329 4330
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
X
Xiaoyu Wang 已提交
4331
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
4332
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
L
Liu Jicong 已提交
4333 4334
        doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX,
                              pChildOp->exprSupp.numOfExprs, 0, NULL);
4335
        rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated);
4336 4337
      }
      taosArrayDestroy(pWins);
5
54liuyao 已提交
4338
      continue;
5
54liuyao 已提交
4339 4340 4341
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      // gap must be 0
5
54liuyao 已提交
4342
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL);
5
54liuyao 已提交
4343 4344 4345 4346 4347
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
        // gap must be 0
5
54liuyao 已提交
4348
        doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL);
4349
        rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated);
5
54liuyao 已提交
4350 4351
      }
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
4352
      removeSessionResults(pStUpdated, pWins);
5
54liuyao 已提交
4353 4354
      taosArrayDestroy(pWins);
      continue;
4355
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4356
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
5
54liuyao 已提交
4357
      continue;
5
54liuyao 已提交
4358
    }
5
54liuyao 已提交
4359

5
54liuyao 已提交
4360 4361 4362 4363
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4364
    // the pDataBlock are always the same one, no need to call this again
4365
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4366 4367 4368 4369 4370 4371
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo));
    if (IS_FINAL_OP(pInfo)) {
      int32_t chIndex = getChildIndex(pBlock);
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
4372 4373
        SOperatorInfo* pChildOp =
            createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
5
54liuyao 已提交
4374
        if (!pChildOp) {
4375
          T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4376 4377 4378
        }
        taosArrayPush(pInfo->pChildren, &pChildOp);
      }
4379
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
4380 4381
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true);
4382
    }
5
54liuyao 已提交
4383
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
4384
    maxTs = TMAX(maxTs, pBlock->info.watermark);
5
54liuyao 已提交
4385
  }
5
54liuyao 已提交
4386 4387

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4388 4389
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
4390

4391
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession,
5
54liuyao 已提交
4392 4393
                     pInfo->ignoreExpiredData, NULL);
  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, NULL);
5
54liuyao 已提交
4394
  copyUpdateResult(pStUpdated, pUpdated);
4395
  removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
5
54liuyao 已提交
4396 4397
  taosHashCleanup(pStUpdated);

4398
  finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4399 4400 4401 4402
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4403
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4404 4405 4406
    return pInfo->pDelRes;
  }
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
4407
  printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4408 4409 4410 4411
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
4412
  void** pIte = NULL;
5
54liuyao 已提交
4413
  while ((pIte = taosHashIterate(pInfo->streamAggSup.pResultRows, pIte)) != NULL) {
4414
    SArray* pWins = (SArray*)(*pIte);
5
54liuyao 已提交
4415 4416 4417 4418 4419 4420 4421 4422
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      SResultWindowInfo* pWin = (SResultWindowInfo*)taosArrayGet(pWins, i);
      pWin->pos.pageId = -1;
      pWin->pos.offset = -1;
    }
  }
  clearDiskbasedBuf(pInfo->streamAggSup.pResultBuf);
4423
  pInfo->streamAggSup.currentPageId = -1;
5
54liuyao 已提交
4424 4425 4426 4427 4428 4429 4430
}

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
  TSKEY                          maxTs = INT64_MIN;
  SExprSupp*                     pSup = &pOperator->exprSupp;
4431

5
54liuyao 已提交
4432 4433
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
4434
  }
L
Liu Jicong 已提交
4435

4436
  {
5
54liuyao 已提交
4437 4438
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
    if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
4439
      printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
4440 4441 4442
      return pBInfo->pRes;
    }

4443 4444
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
H
Haojun Liao 已提交
4445
      printDataBlock(pInfo->pDelRes, "semi session");
5
54liuyao 已提交
4446 4447
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
4448

4449 4450
    if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
      pInfo->returnUpdate = false;
5
54liuyao 已提交
4451
      // process the rest of the data
H
Haojun Liao 已提交
4452
      printDataBlock(pInfo->pUpdateRes, "semi session");
5
54liuyao 已提交
4453 4454
      return pInfo->pUpdateRes;
    }
4455 4456

    if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
4457
      clearFunctionContext(&pOperator->exprSupp);
4458 4459 4460 4461 4462
      // semi interval operator clear disk buffer
      clearStreamSessionOperator(pInfo);
      pOperator->status = OP_EXEC_DONE;
      return NULL;
    }
5
54liuyao 已提交
4463 4464 4465 4466 4467 4468 4469 4470 4471
  }

  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  SOperatorInfo* downstream = pOperator->pDownstream[0];
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
4472
      clearSpecialDataBlock(pInfo->pUpdateRes);
4473
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
4474 4475
      break;
    }
H
Haojun Liao 已提交
4476
    printDataBlock(pBlock, "semi session recv");
5
54liuyao 已提交
4477

5
54liuyao 已提交
4478
    if (pBlock->info.type == STREAM_CLEAR) {
5
54liuyao 已提交
4479
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
H
Haojun Liao 已提交
4480
      doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, START_TS_COLUMN_INDEX, pSup->numOfExprs, 0, pWins);
5
54liuyao 已提交
4481 4482
      removeSessionResults(pStUpdated, pWins);
      taosArrayDestroy(pWins);
H
Haojun Liao 已提交
4483
      copyDataBlock(pInfo->pUpdateRes, pBlock);
4484
      pInfo->returnUpdate = true;
5
54liuyao 已提交
4485
      break;
5
54liuyao 已提交
4486 4487
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      // gap must be 0
4488 4489 4490 4491 4492
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL);
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
      removeSessionResults(pStUpdated, pWins);
      taosArrayDestroy(pWins);
5
54liuyao 已提交
4493
      break;
5
54liuyao 已提交
4494 4495 4496 4497 4498
    } else if (pBlock->info.type == STREAM_GET_ALL) {
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
      continue;
    }

5
54liuyao 已提交
4499 4500 4501 4502
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
5
54liuyao 已提交
4503 4504
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
4505
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, NULL, false);
5
54liuyao 已提交
4506 4507 4508 4509
    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
4510
  pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;
4511

5
54liuyao 已提交
4512
  copyUpdateResult(pStUpdated, pUpdated);
4513
  removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
5
54liuyao 已提交
4514
  taosHashCleanup(pStUpdated);
5
54liuyao 已提交
4515

4516 4517
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4518
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
5
54liuyao 已提交
4519
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
4520 4521 4522

  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
  if (pBInfo->pRes->info.rows > 0) {
H
Haojun Liao 已提交
4523
    printDataBlock(pBInfo->pRes, "semi session");
5
54liuyao 已提交
4524 4525 4526
    return pBInfo->pRes;
  }

4527 4528
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
H
Haojun Liao 已提交
4529
    printDataBlock(pInfo->pDelRes, "semi session");
5
54liuyao 已提交
4530 4531
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
4532

4533 4534
  if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
    pInfo->returnUpdate = false;
5
54liuyao 已提交
4535
    // process the rest of the data
H
Haojun Liao 已提交
4536
    printDataBlock(pInfo->pUpdateRes, "semi session");
5
54liuyao 已提交
4537 4538
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
4539 4540 4541

  pOperator->status = OP_EXEC_DONE;
  return NULL;
5
54liuyao 已提交
4542
}
4543

4544 4545
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                       SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
4546 4547
  int32_t        code = TSDB_CODE_OUT_OF_MEMORY;
  SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo);
4548 4549 4550
  if (pOperator == NULL) {
    goto _error;
  }
4551
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
5
54liuyao 已提交
4552 4553 4554 4555 4556 4557

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
    pInfo->isFinal = true;
    pOperator->name = "StreamSessionFinalAggOperator";
  } else {
    pInfo->isFinal = false;
H
Haojun Liao 已提交
4558
    pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
4559 4560 4561
    blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
    pOperator->name = "StreamSessionSemiAggOperator";
    pOperator->fpSet =
4562 4563
        createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL,
                            destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
4564 4565 4566 4567 4568
  }
  pOperator->operatorType = pPhyNode->type;
  if (numOfChild > 0) {
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
    for (int32_t i = 0; i < numOfChild; i++) {
4569
      SOperatorInfo* pChild = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
5
54liuyao 已提交
4570 4571 4572 4573
      if (pChild == NULL) {
        goto _error;
      }
      taosArrayPush(pInfo->pChildren, &pChild);
4574 4575 4576 4577 4578 4579
    }
  }
  return pOperator;

_error:
  if (pInfo != NULL) {
4580
    destroyStreamSessionAggOperatorInfo(pInfo);
4581 4582 4583 4584 4585 4586
  }

  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5
54liuyao 已提交
4587

4588
void destroyStreamStateOperatorInfo(void* param) {
X
Xiaoyu Wang 已提交
4589
  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;
4590
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
4591
  destroyStateStreamAggSupporter(&pInfo->streamAggSup);
5
54liuyao 已提交
4592 4593 4594 4595
  cleanupGroupResInfo(&pInfo->groupResInfo);
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
4596
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
5
54liuyao 已提交
4597
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
4598
      destroyStreamSessionAggOperatorInfo(pChInfo);
5
54liuyao 已提交
4599 4600 4601 4602
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
5
54liuyao 已提交
4603 4604 4605 4606
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  taosHashCleanup(pInfo->pSeDeleted);
  destroySqlFunctionCtx(pInfo->pDummyCtx, 0);
D
dapan1121 已提交
4607 4608

  taosMemoryFreeClear(param);
5
54liuyao 已提交
4609 4610 4611 4612 4613 4614 4615
}

int64_t getStateWinTsKey(void* data, int32_t index) {
  SStateWindowInfo* pStateWin = taosArrayGet(data, index);
  return pStateWin->winInfo.win.ekey;
}

X
Xiaoyu Wang 已提交
4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627
SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4628 4629 4630 4631 4632 4633 4634 4635
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayPush(pWinInfos, &win);
}

X
Xiaoyu Wang 已提交
4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647
SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayInsert(pWinInfos, index, &win);
}

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {
  if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) {
    return true;
  }
  return false;
}

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {
  return pKeyData && compareVal(pKeyData, &pWin->stateKey);
}

4667 4668 4669
SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
X
Xiaoyu Wang 已提交
4670 4671
  int32_t           size = taosArrayGetSize(pWinInfos);
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index + 1;
      return pWin;
    }
  }
  *pIndex = 0;
  return NULL;
}

4692 4693
SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
                                 SColumn* pCol, int32_t* pIndex) {
4694 4695
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
5
54liuyao 已提交
4696 4697 4698 4699 4700
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    *pIndex = 0;
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
X
Xiaoyu Wang 已提交
4701
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
  *pIndex = index + 1;
  return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol);
}

H
Haojun Liao 已提交
4735 4736 4737
int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, uint64_t groupId,
                              SColumnInfoData* pKeyCol, int32_t rows, int32_t start, bool* allEqual,
                              SHashObj* pSeDeleted) {
5
54liuyao 已提交
4738 4739 4740 4741 4742
  *allEqual = true;
  SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex);
  for (int32_t i = start; i < rows; ++i) {
    char* pKeyData = colDataGetData(pKeyCol, i);
    if (!isTsInWindow(pWinInfo, pTs[i])) {
X
Xiaoyu Wang 已提交
4743
      if (isEqualStateKey(pWinInfo, pKeyData)) {
5
54liuyao 已提交
4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756
        int32_t size = taosArrayGetSize(pWinInfos);
        if (winIndex + 1 < size) {
          SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1);
          // ts belongs to the next window
          if (pTs[i] >= pNextWin->winInfo.win.skey) {
            return i - start;
          }
        }
      } else {
        return i - start;
      }
    }
    if (pWinInfo->winInfo.win.skey > pTs[i]) {
H
Haojun Liao 已提交
4757 4758
      if (pSeDeleted && pWinInfo->winInfo.isOutput) {
        SWinKey res = {.ts = pWinInfo->winInfo.win.skey, .groupId = groupId};
4759
        taosHashPut(pSeDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
5
54liuyao 已提交
4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771
        pWinInfo->winInfo.isOutput = false;
      }
      pWinInfo->winInfo.win.skey = pTs[i];
    }
    pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]);
    if (!isEqualStateKey(pWinInfo, pKeyData)) {
      *allEqual = false;
    }
  }
  return rows - start;
}

H
Haojun Liao 已提交
4772 4773 4774 4775
static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SHashObj* pSeUpdated,
                                SHashObj* pSeDeleted) {
  SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupColInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
X
Xiaoyu Wang 已提交
4776 4777 4778
  TSKEY*           tsCol = (TSKEY*)pTsColInfo->pData;
  bool             allEqual = false;
  int32_t          step = 1;
H
Haojun Liao 已提交
4779
  uint64_t*        gpCol = (uint64_t*)pGroupColInfo->pData;
5
54liuyao 已提交
4780
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
4781
    int32_t           winIndex = 0;
H
Haojun Liao 已提交
4782
    SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], gpCol[i], &winIndex);
5
54liuyao 已提交
4783 4784 4785
    if (!pCurWin) {
      continue;
    }
H
Haojun Liao 已提交
4786
    updateSessionWindowInfo(&pCurWin->winInfo, tsCol, NULL, 0, pBlock->info.rows, i, 0, NULL);
5
54liuyao 已提交
4787
    taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
5
54liuyao 已提交
4788
    deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
5
54liuyao 已提交
4789 4790 4791
  }
}

X
Xiaoyu Wang 已提交
4792 4793 4794
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated,
                                 SHashObj* pStDeleted) {
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
4795
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4796
  bool                         masterScan = true;
4797
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
X
Xiaoyu Wang 已提交
4798 4799 4800 4801 4802 4803 4804
  int64_t                      groupId = pSDataBlock->info.groupId;
  int64_t                      code = TSDB_CODE_SUCCESS;
  int32_t                      step = 1;
  bool                         ascScan = true;
  TSKEY*                       tsCols = NULL;
  SResultRow*                  pResult = NULL;
  int32_t                      winRows = 0;
5
54liuyao 已提交
4805
  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
4806 4807
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
4808
  } else {
X
Xiaoyu Wang 已提交
4809
    return;
5
54liuyao 已提交
4810
  }
L
Liu Jicong 已提交
4811

5
54liuyao 已提交
4812
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
4813
  blockDataEnsureCapacity(pAggSup->pScanBlock, pSDataBlock->info.rows);
L
Liu Jicong 已提交
4814
  SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
X
Xiaoyu Wang 已提交
4815
  for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
5
54liuyao 已提交
4816
    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
4817 4818 4819
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
4820 4821 4822
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
    bool              allEqual = true;
H
Haojun Liao 已提交
4823 4824 4825
    SStateWindowInfo* pCurWin = getStateWindow(pAggSup, tsCols[i], groupId, pKeyData, &pInfo->stateCol, &winIndex);
    winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, groupId, pKeyColInfo, pSDataBlock->info.rows,
                                    i, &allEqual, pStDeleted);
5
54liuyao 已提交
4826
    if (!allEqual) {
4827
      uint64_t uid = 0;
L
Liu Jicong 已提交
4828
      appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, &uid, &groupId);
5
54liuyao 已提交
4829
      taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
5
54liuyao 已提交
4830
      deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
5
54liuyao 已提交
4831 4832
      continue;
    }
4833
    code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
4834
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
4835
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4836 4837 4838
    }
    pCurWin->winInfo.isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
H
Haojun Liao 已提交
4839 4840
      SWinKey value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId};
      code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
5
54liuyao 已提交
4841
      if (code != TSDB_CODE_SUCCESS) {
4842
        T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5
54liuyao 已提交
4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853
      }
      pCurWin->winInfo.isOutput = true;
    }
  }
}

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

4854
  SExprSupp*                   pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4855
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4856
  SOptrBasicInfo*              pBInfo = &pInfo->binfo;
L
Liu Jicong 已提交
4857
  int64_t                      maxTs = INT64_MIN;
5
54liuyao 已提交
4858 4859 4860
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4861
      printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4862 4863
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
4864
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
4865
    if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
5
54liuyao 已提交
4866 4867
      doSetOperatorCompleted(pOperator);
    }
5
54liuyao 已提交
4868
    printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4869 4870 4871
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
4872 4873
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
4874
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
4875
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
4876 4877 4878 4879 4880
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4881
    printDataBlock(pBlock, "single state recv");
4882

5
54liuyao 已提交
4883
    if (pBlock->info.type == STREAM_CLEAR) {
H
Haojun Liao 已提交
4884
      doClearStateWindows(&pInfo->streamAggSup, pBlock, pSeUpdated, pInfo->pSeDeleted);
5
54liuyao 已提交
4885
      continue;
4886
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
4887
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
5
54liuyao 已提交
4888
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo);
4889
      copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
4890
      removeSessionResults(pSeUpdated, pWins);
4891 4892
      taosArrayDestroy(pWins);
      continue;
4893
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4894
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState);
5
54liuyao 已提交
4895
      continue;
5
54liuyao 已提交
4896
    }
4897

5
54liuyao 已提交
4898 4899 4900 4901
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4902
    // the pDataBlock are always the same one, no need to call this again
4903
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4904
    doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
4905
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
4906
  }
4907
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4908 4909
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
X
Xiaoyu Wang 已提交
4910

4911
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState,
5
54liuyao 已提交
4912 4913
                     pInfo->ignoreExpiredData, destroyStateWinInfo);
  // closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, destroyStateWinInfo);
5
54liuyao 已提交
4914
  copyUpdateResult(pSeUpdated, pUpdated);
5
54liuyao 已提交
4915 4916
  taosHashCleanup(pSeUpdated);

4917 4918
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4919 4920 4921 4922
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4923
    printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4924 4925
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
4926
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
4927
  printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4928 4929 4930
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

4931 4932 4933 4934
int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo));
}

X
Xiaoyu Wang 已提交
4935 4936 4937 4938 4939 4940
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
  SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
4941
  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
4942

X
Xiaoyu Wang 已提交
4943 4944
  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
4945 4946 4947 4948
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

4949 4950
  SExprSupp* pSup = &pOperator->exprSupp;

X
Xiaoyu Wang 已提交
4951
  int32_t    numOfCols = 0;
5
54liuyao 已提交
4952 4953 4954
  SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
4955
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
4956 4957 4958 4959 4960 4961 4962 4963 4964
  if (pStateNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

4965
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
X
Xiaoyu Wang 已提交
4966 4967
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
5
54liuyao 已提交
4968 4969
      .calTrigger = pStateNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
4970
      .minTs = INT64_MAX,
X
Xiaoyu Wang 已提交
4971
  };
5
54liuyao 已提交
4972
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
4973

4974
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
4975 4976 4977 4978
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

4979
  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
4980 4981 4982 4983 4984 4985 4986 4987 4988
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }

4989
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
4990 4991 4992 4993 4994
  pInfo->primaryTsIndex = tsSlotId;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
H
Haojun Liao 已提交
4995
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
5
54liuyao 已提交
4996
  pInfo->pChildren = NULL;
5
54liuyao 已提交
4997
  pInfo->ignoreExpiredData = pStateNode->window.igExpired;
5
54liuyao 已提交
4998 4999

  pOperator->name = "StreamStateAggOperator";
5000
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
5
54liuyao 已提交
5001 5002 5003 5004
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
5005 5006
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL,
                                         destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
L
Liu Jicong 已提交
5007 5008
  initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType,
                 pInfo->primaryTsIndex);
5
54liuyao 已提交
5009 5010 5011 5012 5013 5014 5015
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
  return pOperator;

_error:
5016
  destroyStreamStateOperatorInfo(pInfo);
5
54liuyao 已提交
5017 5018 5019 5020
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5021

5022
void destroyMAIOperatorInfo(void* param) {
5023
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param;
5024
  destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo);
D
dapan1121 已提交
5025
  taosMemoryFreeClear(param);
5026 5027
}

5028
static SResultRow* doSetSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, SAggSupporter* pSup) {
H
Haojun Liao 已提交
5029 5030
  SResultRow* pResult = getNewResultRow(pSup->pResultBuf, &pSup->currentPageId, pSup->resultRowSize);
  pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
5031 5032
  return pResult;
}
5033

5034 5035 5036 5037 5038 5039 5040 5041
static int32_t setSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, SResultRow** pResult,
                                       SExprSupp* pExprSup, SAggSupporter* pAggSup) {
  if (*pResult == NULL) {
    *pResult = doSetSingleOutputTupleBuf(pResultRowInfo, pAggSup);
    if (*pResult == NULL) {
      return terrno;
    }
  }
5042

5043
  // set time window for current result
5044 5045
  (*pResult)->win = (*win);
  setResultRowInitCtx((*pResult), pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
5046
  return TSDB_CODE_SUCCESS;
5047 5048
}

5049
static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
5050
                                          SSDataBlock* pBlock, SSDataBlock* pResultBlock) {
5051
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
D
dapan1121 已提交
5052
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
5053 5054

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
5055
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
5056
  SInterval*     pInterval = &iaInfo->interval;
5057

5
54liuyao 已提交
5058 5059
  int32_t  startPos = 0;
  int64_t* tsCols = extractTsCol(pBlock, iaInfo);
5060

5061 5062
  TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols);

5063 5064
  // there is an result exists
  if (miaInfo->curTs != INT64_MIN) {
5065
    if (ts != miaInfo->curTs) {
5066
      finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo);
5067
      resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow));
5068
      miaInfo->curTs = ts;
5069
    }
5070 5071
  } else {
    miaInfo->curTs = ts;
5072 5073 5074
  }

  STimeWindow win = {0};
5075
  win.skey = miaInfo->curTs;
5076
  win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
5077

5
54liuyao 已提交
5078
  int32_t ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
5079 5080
  if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) {
    T_LONG_JMP(pTaskInfo->env, ret);
5081 5082
  }

5083 5084
  int32_t currPos = startPos;

5085
  STimeWindow currWin = win;
5086
  while (++currPos < pBlock->info.rows) {
5087
    if (tsCols[currPos] == miaInfo->curTs) {
5088
      continue;
5089 5090 5091
    }

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
5092
    doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
5093
                     pBlock->info.rows, pSup->numOfExprs);
5094

5095
    finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo);
5096
    resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow));
5097
    miaInfo->curTs = tsCols[currPos];
5098

5099
    currWin.skey = miaInfo->curTs;
5100
    currWin.ekey = taosTimeAdd(currWin.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
5101 5102

    startPos = currPos;
5
54liuyao 已提交
5103
    ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
5104 5105
    if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) {
      T_LONG_JMP(pTaskInfo->env, ret);
5106
    }
5107 5108

    miaInfo->curTs = currWin.skey;
5109
  }
5110

5111
  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
5112
  doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
5113
                   pBlock->info.rows, pSup->numOfExprs);
5114 5115
}

5116 5117 5118 5119
static void cleanupAfterGroupResultGen(SMergeAlignedIntervalAggOperatorInfo* pMiaInfo, SSDataBlock* pRes) {
  pRes->info.groupId = pMiaInfo->groupId;
  pMiaInfo->curTs = INT64_MIN;
  pMiaInfo->groupId = 0;
5120 5121
}

5122
static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
S
shenglian zhou 已提交
5123
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
5124

5125 5126
  SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info;
  SIntervalAggOperatorInfo*             pIaInfo = pMiaInfo->intervalAggOperatorInfo;
5127

5128 5129 5130 5131 5132
  SExprSupp*      pSup = &pOperator->exprSupp;
  SSDataBlock*    pRes = pIaInfo->binfo.pRes;
  SResultRowInfo* pResultRowInfo = &pIaInfo->binfo.resultRowInfo;
  SOperatorInfo*  downstream = pOperator->pDownstream[0];
  int32_t         scanFlag = MAIN_SCAN;
5133

5134 5135
  while (1) {
    SSDataBlock* pBlock = NULL;
5136
    if (pMiaInfo->prefetchedBlock == NULL) {
5137 5138
      pBlock = downstream->fpSet.getNextFn(downstream);
    } else {
5139 5140
      pBlock = pMiaInfo->prefetchedBlock;
      pMiaInfo->prefetchedBlock = NULL;
5141

5142
      pMiaInfo->groupId = pBlock->info.groupId;
5143
    }
5144

5145
    // no data exists, all query processing is done
5146
    if (pBlock == NULL) {
5147 5148 5149
      // close last unclosed time window
      if (pMiaInfo->curTs != INT64_MIN) {
        finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo);
5150 5151
        resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow));
        cleanupAfterGroupResultGen(pMiaInfo, pRes);
5152
      }
5153

5154 5155
      doSetOperatorCompleted(pOperator);
      break;
5156
    }
5157

H
Haojun Liao 已提交
5158 5159 5160 5161 5162 5163 5164 5165 5166
    if (pMiaInfo->groupId == 0) {
      if (pMiaInfo->groupId != pBlock->info.groupId) {
        pMiaInfo->groupId = pBlock->info.groupId;
      }
    } else {
      if (pMiaInfo->groupId != pBlock->info.groupId) {
        // if there are unclosed time window, close it firstly.
        ASSERT(pMiaInfo->curTs != INT64_MIN);
        finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo);
5167
        resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow));
H
Haojun Liao 已提交
5168

5169 5170
        pMiaInfo->prefetchedBlock = pBlock;
        cleanupAfterGroupResultGen(pMiaInfo, pRes);
H
Haojun Liao 已提交
5171
        break;
5
54liuyao 已提交
5172
      } else {
H
Haojun Liao 已提交
5173 5174
        // continue
      }
5175
    }
5176

5177 5178 5179
    getTableScanInfo(pOperator, &pIaInfo->inputOrder, &scanFlag);
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pIaInfo->inputOrder, scanFlag, true);
    doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes);
5180

5181
    doFilter(pMiaInfo->pCondition, pRes, NULL);
5182 5183 5184
    if (pRes->info.rows >= pOperator->resultInfo.capacity) {
      break;
    }
5185
  }
5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200
}

static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info;
  SIntervalAggOperatorInfo*             iaInfo = pMiaInfo->intervalAggOperatorInfo;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);

  if (iaInfo->binfo.mergeResultBlock) {
dengyihao's avatar
dengyihao 已提交
5201
    while (1) {
5202
      if (pOperator->status == OP_EXEC_DONE) {
5203 5204
        break;
      }
5205

5206
      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
5207 5208 5209
        break;
      }

5210 5211 5212 5213
      doMergeAlignedIntervalAgg(pOperator);
    }
  } else {
    doMergeAlignedIntervalAgg(pOperator);
5214 5215 5216 5217 5218 5219 5220
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

5221
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode,
5222
                                                      SExecTaskInfo* pTaskInfo) {
5223
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo));
5224
  SOperatorInfo*                        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5225 5226 5227 5228
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

D
dapan1121 已提交
5229 5230 5231 5232 5233
  miaInfo->intervalAggOperatorInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
  if (miaInfo->intervalAggOperatorInfo == NULL) {
    goto _error;
  }

5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244
  int32_t      num = 0;
  SExprInfo*   pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pNode->window.node.pOutputDataBlockDesc);

  SInterval interval = {.interval = pNode->interval,
                        .sliding = pNode->sliding,
                        .intervalUnit = pNode->intervalUnit,
                        .slidingUnit = pNode->slidingUnit,
                        .offset = pNode->offset,
                        .precision = ((SColumnNode*)pNode->window.pTspk)->node.resType.precision};

D
dapan1121 已提交
5245
  SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
5246
  SExprSupp*                pSup = &pOperator->exprSupp;
5247

5248
  miaInfo->pCondition = pNode->window.node.pConditions;
L
Liu Jicong 已提交
5249 5250 5251 5252 5253
  miaInfo->curTs = INT64_MIN;
  iaInfo->win = pTaskInfo->window;
  iaInfo->inputOrder = TSDB_ORDER_ASC;
  iaInfo->interval = interval;
  iaInfo->execModel = pTaskInfo->execModel;
5254 5255
  iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId;
  iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock;
5256 5257

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
5258
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5259

5260
  int32_t code = initAggInfo(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
5261 5262 5263
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
5264

5265
  initBasicInfo(&iaInfo->binfo, pResBlock);
5266
  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
5267

5268
  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, iaInfo);
5269
  if (iaInfo->timeWindowInterpo) {
5270
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
5271 5272
  }

5273
  initResultRowInfo(&iaInfo->binfo.resultRowInfo);
5274
  blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity);
5275

L
Liu Jicong 已提交
5276
  pOperator->name = "TimeMergeAlignedIntervalAggOperator";
5277
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL;
L
Liu Jicong 已提交
5278 5279 5280 5281
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = miaInfo;
5282

5283
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL,
5284
                                         destroyMAIOperatorInfo, NULL, NULL, NULL);
5285 5286 5287 5288 5289 5290 5291 5292 5293

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5294
  destroyMAIOperatorInfo(miaInfo);
5295 5296 5297 5298
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5299 5300 5301 5302 5303

//=====================================================================================================================
// merge interval operator
typedef struct SMergeIntervalAggOperatorInfo {
  SIntervalAggOperatorInfo intervalAggOperatorInfo;
L
Liu Jicong 已提交
5304 5305 5306 5307 5308 5309
  SList*                   groupIntervals;
  SListIter                groupIntervalsIter;
  bool                     hasGroupId;
  uint64_t                 groupId;
  SSDataBlock*             prefetchedBlock;
  bool                     inputBlocksFinished;
5310 5311
} SMergeIntervalAggOperatorInfo;

S
slzhou 已提交
5312
typedef struct SGroupTimeWindow {
L
Liu Jicong 已提交
5313
  uint64_t    groupId;
S
slzhou 已提交
5314 5315 5316
  STimeWindow window;
} SGroupTimeWindow;

5317
void destroyMergeIntervalOperatorInfo(void* param) {
5318
  SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
S
slzhou 已提交
5319
  tdListFree(miaInfo->groupIntervals);
5320
  destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo);
5321

D
dapan1121 已提交
5322
  taosMemoryFreeClear(param);
5323 5324
}

L
Liu Jicong 已提交
5325 5326
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win,
                                    SSDataBlock* pResultBlock) {
5327 5328 5329
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
5330
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5331 5332 5333
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId);
L
Liu Jicong 已提交
5334 5335
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(
      iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
5336
  ASSERT(p1 != NULL);
5
54liuyao 已提交
5337
  //  finalizeResultRows(iaInfo->aggSup.pResultBuf, p1, pResultBlock, pTaskInfo);
5338
  tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
5339 5340 5341
  return TSDB_CODE_SUCCESS;
}

5342 5343 5344 5345
static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock,
                                        STimeWindow* newWin) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
5346
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5347

S
slzhou 已提交
5348 5349
  SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin};
  tdListAppend(miaInfo->groupIntervals, &groupTimeWindow);
5350

S
slzhou 已提交
5351 5352 5353 5354 5355
  SListIter iter = {0};
  tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD);
  SListNode* listNode = NULL;
  while ((listNode = tdListNext(&iter)) != NULL) {
    SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
L
Liu Jicong 已提交
5356
    if (prevGrpWin->groupId != tableGroupId) {
S
slzhou 已提交
5357 5358
      continue;
    }
5359

S
slzhou 已提交
5360
    STimeWindow* prevWin = &prevGrpWin->window;
H
Haojun Liao 已提交
5361
    if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) {
5
54liuyao 已提交
5362
      //      finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
S
slzhou 已提交
5363 5364
      tdListPopNode(miaInfo->groupIntervals, listNode);
    }
5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381
  }

  return 0;
}

static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                                   int32_t scanFlag, SSDataBlock* pResultBlock) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*     pExprSup = &pOperatorInfo->exprSupp;

  int32_t     startPos = 0;
  int32_t     numOfOutput = pExprSup->numOfExprs;
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
5382
  bool        ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5383 5384 5385
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

5386 5387
  STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval,
                                        iaInfo->inputOrder);
5388 5389 5390 5391 5392

  int32_t ret =
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                             numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
5393
    T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5394 5395 5396 5397
  }

  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
5398
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5399 5400 5401 5402
  ASSERT(forwardRows > 0);

  // prev time window not interpolation yet.
  if (iaInfo->timeWindowInterpo) {
5403
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult, tableGroupId);
5404 5405 5406 5407 5408 5409
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);

    // restore current time window
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                                 numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS) {
5410
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5411 5412 5413 5414 5415 5416 5417
    }

    // window start key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup);
  }

  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true);
5418 5419
  doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
                   pBlock->info.rows, numOfOutput);
5420 5421 5422 5423 5424 5425 5426 5427
  doCloseWindow(pResultRowInfo, iaInfo, pResult);

  // output previous interval results after this interval (&win) is closed
  outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win);

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = forwardRows - 1 + startPos;
5428 5429
    startPos =
        getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->inputOrder);
5430 5431 5432 5433 5434 5435 5436 5437 5438
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
    int32_t code =
        setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                               pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
5439
      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
5440 5441 5442 5443
    }

    ekey = ascScan ? nextWin.ekey : nextWin.skey;
    forwardRows =
5444
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5445 5446 5447 5448 5449

    // window start(end) key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup);

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true);
5450 5451
    doApplyFunctions(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     pBlock->info.rows, numOfOutput);
5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491
    doCloseWindow(pResultRowInfo, iaInfo, pResult);

    // output previous interval results after this interval (&nextWin) is closed
    outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin);
  }

  if (iaInfo->timeWindowInterpo) {
    saveDataBlockLastRow(iaInfo->pPrevValues, pBlock, iaInfo->pInterpCols);
  }
}

static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExprSupp*                     pExpSupp = &pOperator->exprSupp;

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);

  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
        miaInfo->groupId = pBlock->info.groupId;
        miaInfo->prefetchedBlock = NULL;
      }

      if (pBlock == NULL) {
S
slzhou 已提交
5492
        tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD);
5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504
        miaInfo->inputBlocksFinished = true;
        break;
      }

      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
        miaInfo->groupId = pBlock->info.groupId;
      } else if (miaInfo->groupId != pBlock->info.groupId) {
        miaInfo->prefetchedBlock = pBlock;
        break;
      }

5505 5506
      getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag);
      setInputDataBlock(pOperator, pExpSupp->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true);
5507 5508 5509 5510 5511 5512 5513 5514
      doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);

      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
        break;
      }
    }

    pRes->info.groupId = miaInfo->groupId;
5515 5516 5517
  }

  if (miaInfo->inputBlocksFinished) {
S
slzhou 已提交
5518
    SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter);
5519

S
slzhou 已提交
5520 5521
    if (listNode != NULL) {
      SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data);
5
54liuyao 已提交
5522
      //      finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
S
slzhou 已提交
5523
      pRes->info.groupId = grpWin->groupId;
5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535
    }
  }

  if (pRes->info.rows == 0) {
    doSetOperatorCompleted(pOperator);
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

5536 5537 5538
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode,
                                               SExecTaskInfo* pTaskInfo) {
  SMergeIntervalAggOperatorInfo* pMergeIntervalInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
5539
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5540
  if (pMergeIntervalInfo == NULL || pOperator == NULL) {
5541 5542 5543
    goto _error;
  }

5544 5545 5546 5547 5548 5549 5550 5551 5552 5553
  int32_t      num = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
  SSDataBlock* pResBlock = createResDataBlock(pIntervalPhyNode->window.node.pOutputDataBlockDesc);

  SInterval interval = {.interval = pIntervalPhyNode->interval,
                        .sliding = pIntervalPhyNode->sliding,
                        .intervalUnit = pIntervalPhyNode->intervalUnit,
                        .slidingUnit = pIntervalPhyNode->slidingUnit,
                        .offset = pIntervalPhyNode->offset,
                        .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
5554

5555
  pMergeIntervalInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow));
5556

5557
  SIntervalAggOperatorInfo* pIntervalInfo = &pMergeIntervalInfo->intervalAggOperatorInfo;
L
Liu Jicong 已提交
5558
  pIntervalInfo->win = pTaskInfo->window;
5559
  pIntervalInfo->inputOrder = TSDB_ORDER_ASC;
L
Liu Jicong 已提交
5560 5561
  pIntervalInfo->interval = interval;
  pIntervalInfo->execModel = pTaskInfo->execModel;
5562 5563
  pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock;
  pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
5564 5565 5566 5567

  SExprSupp* pExprSupp = &pOperator->exprSupp;

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
5568
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5569

5570
  int32_t code = initAggInfo(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
5571 5572 5573
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
5574

5575 5576
  initBasicInfo(&pIntervalInfo->binfo, pResBlock);
  initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win);
5577

5578 5579
  pIntervalInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, num, pIntervalInfo);
  if (pIntervalInfo->timeWindowInterpo) {
5580
    pIntervalInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo));
5581
    if (pIntervalInfo->binfo.resultRowInfo.openWindow == NULL) {
5582 5583 5584 5585
      goto _error;
    }
  }

5586
  initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo);
5587

L
Liu Jicong 已提交
5588
  pOperator->name = "TimeMergeIntervalAggOperator";
5589
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL;
L
Liu Jicong 已提交
5590 5591 5592 5593
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pMergeIntervalInfo;
5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL,
                                         destroyMergeIntervalOperatorInfo, NULL, NULL, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5606
  destroyMergeIntervalOperatorInfo(pMergeIntervalInfo);
5607 5608 5609
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
5610
}
5611 5612 5613

static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
5614 5615
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
  int64_t                      maxTs = INT64_MIN;
5
54liuyao 已提交
5616
  int64_t                      minTs = INT64_MAX;
5617
  SExprSupp*                   pSup = &pOperator->exprSupp;
5618 5619 5620 5621 5622 5623 5624 5625

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
5626
      printDataBlock(pInfo->pDelRes, "single interval delete");
5627 5628 5629
      return pInfo->pDelRes;
    }

5630
    doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
5631 5632 5633
    if (pInfo->binfo.pRes->info.rows > 0) {
      printDataBlock(pInfo->binfo.pRes, "single interval");
      return pInfo->binfo.pRes;
5634
    }
5635 5636
    deleteIntervalDiscBuf(pInfo->pState, NULL, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, &pInfo->interval,
                          &pInfo->delKey);
5
54liuyao 已提交
5637 5638
    doSetOperatorCompleted(pOperator);
    return NULL;
5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];

  SArray*    pUpdated = taosArrayInit(4, POINTER_BYTES);  // SResKeyPos
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*  pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
    printDataBlock(pBlock, "single interval recv");

    if (pBlock->info.type == STREAM_CLEAR) {
5
54liuyao 已提交
5655
      doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, NULL, NULL);
5656 5657
      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
      continue;
5658
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
5
54liuyao 已提交
5659 5660
      doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pInfo->pDelWins,
                      pUpdatedMap);
5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685
      continue;
    } else if (pBlock->info.type == STREAM_GET_ALL) {
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
      continue;
    }

    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {
      // set input version
      pTaskInfo->version = pBlock->info.version;
    }

    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }

    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
    // caller. Note that all the time window are not close till now.
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
    if (pInfo->invertible) {
      setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type);
    }

    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
5686
    minTs = TMIN(minTs, pBlock->info.window.skey);
5687
    doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
5688 5689
  }
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
5690
  pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs);
5691
  pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
5692 5693
  closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap,
                            pOperator);
5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704

  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosArraySort(pUpdated, resultrowComparAsc);

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
  taosHashCleanup(pUpdatedMap);
5
54liuyao 已提交
5705

5706 5707
  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
5708
    printDataBlock(pInfo->pDelRes, "single interval delete");
5709 5710 5711
    return pInfo->pDelRes;
  }

5712
  doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
5
54liuyao 已提交
5713 5714 5715 5716 5717 5718
  if (pInfo->binfo.pRes->info.rows > 0) {
    printDataBlock(pInfo->binfo.pRes, "single interval");
    return pInfo->binfo.pRes;
  }

  return NULL;
5719 5720 5721 5722 5723
}

SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
5724
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5725 5726 5727 5728 5729
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

5730 5731
  int32_t    numOfCols = 0;
  SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5732 5733
  ASSERT(numOfCols > 0);
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745
  SInterval    interval = {
         .interval = pIntervalPhyNode->interval,
         .sliding = pIntervalPhyNode->sliding,
         .intervalUnit = pIntervalPhyNode->intervalUnit,
         .slidingUnit = pIntervalPhyNode->slidingUnit,
         .offset = pIntervalPhyNode->offset,
         .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,
  };
  STimeWindowAggSupp twAggSupp = {
      .waterMark = pIntervalPhyNode->window.watermark,
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
5
54liuyao 已提交
5746
      .minTs = INT64_MAX,
5
54liuyao 已提交
5747
      .deleteMark = INT64_MAX,
5748
  };
5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764
  ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY);
  pOperator->pTaskInfo = pTaskInfo;
  pInfo->interval = interval;
  pInfo->twAggSup = twAggSupp;
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
  pInfo->isFinal = false;

  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

5765
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
5766 5767
  initResultSizeInfo(&pOperator->resultInfo, 4096);
  SExprSupp* pSup = &pOperator->exprSupp;
5768 5769
  size_t     keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  int32_t    code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
5770 5771 5772 5773 5774 5775 5776 5777 5778
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  initBasicInfo(&pInfo->binfo, pResBlock);
  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
5779
  pInfo->invertible = false;
5780 5781 5782 5783 5784
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
  pInfo->delIndex = 0;
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
  initResultRowInfo(&pInfo->binfo.resultRowInfo);

5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800
  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
  *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
  streamStateSetNumber(pInfo->pState, -1);

  pInfo->pUpdateRes = NULL;
  pInfo->returnUpdate = false;
  pInfo->pPhyNode = NULL;  // create new child
  pInfo->pPullDataMap = NULL;
  pInfo->pPullWins = NULL;  // SPullWindowInfo
  pInfo->pullIndex = 0;
  pInfo->pPullDataRes = NULL;
  pInfo->isFinal = false;
  pInfo->pChildren = NULL;
  pInfo->delKey.ts = INT64_MAX;
  pInfo->delKey.groupId = 0;

5801 5802 5803 5804 5805
  pOperator->name = "StreamIntervalOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
5806
  pOperator->fpSet =
5807
      createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
5808
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5809

5
54liuyao 已提交
5810
  initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
5811 5812 5813 5814 5815 5816 5817 5818
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5819
  destroyStreamFinalIntervalOperatorInfo(pInfo);
5820 5821 5822 5823
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}