timewindowoperator.c 211.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15
#include "executorimpl.h"
X
Xiaoyu Wang 已提交
16
#include "function.h"
5
54liuyao 已提交
17
#include "functionMgt.h"
5
54liuyao 已提交
18
#include "tcompare.h"
L
Liu Jicong 已提交
19
#include "tdatablock.h"
H
Haojun Liao 已提交
20
#include "tfill.h"
21
#include "ttime.h"
22 23 24 25 26 27

typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;

5
54liuyao 已提交
28 29
#define IS_FINAL_OP(op) ((op)->isFinal)

5
54liuyao 已提交
30 31 32 33 34 35 36
typedef struct SWinRes {
  TSKEY    ts;
  uint64_t groupId;
} SWinRes;

typedef struct SPullWindowInfo {
  STimeWindow window;
L
Liu Jicong 已提交
37
  uint64_t    groupId;
5
54liuyao 已提交
38 39
} SPullWindowInfo;

5
54liuyao 已提交
40
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
5
54liuyao 已提交
41

42 43 44 45 46
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);

static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult);
static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult);

H
Haojun Liao 已提交
47 48 49 50 51 52 53 54 55
///*
// * There are two cases to handle:
// *
// * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including
// * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey.
// * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be
// *    merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there
// *    is a previous result generated or not.
// */
56
// static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) {
H
Haojun Liao 已提交
57 58
//  // do nothing
//}
59

X
Xiaoyu Wang 已提交
60
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }
61 62 63

static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
64
                                      int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
65 66 67 68 69 70 71 72 73 74 75
                                      SExecTaskInfo* pTaskInfo) {
  SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                  masterscan, tableGroupId, pTaskInfo, true, pAggSup);

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // set time window for current result
  pResultRow->win = (*win);
76

77
  *pResult = pResultRow;
78
  setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
79

80 81 82 83 84 85 86 87 88 89 90 91 92
  return TSDB_CODE_SUCCESS;
}

static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) {
  int64_t* ts = (int64_t*)pColData->pData;
  int32_t  delta = includeEndpoint ? 1 : 0;

  int64_t duration = pWin->ekey - pWin->skey + delta;
  ts[2] = duration;            // set the duration
  ts[3] = pWin->skey;          // window start key
  ts[4] = pWin->ekey + delta;  // window end key
}

93
static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) {
94 95 96
  pRowSup->win.ekey = ts;
  pRowSup->prevTs = ts;
  pRowSup->numOfRows += 1;
97
  pRowSup->groupId = groupId;
98 99
}

100
static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex, uint64_t groupId) {
101 102 103
  pRowSup->startRowIndex = rowIndex;
  pRowSup->numOfRows = 0;
  pRowSup->win.skey = tsList[rowIndex];
104
  pRowSup->groupId = groupId;
105 106 107 108
}

static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
                                                   int16_t pos, int16_t order, int64_t* pData) {
109
  int32_t forwardRows = 0;
110 111 112 113

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
114
      forwardRows = end;
115 116

      if (pData[end + pos] == ekey) {
117
        forwardRows += 1;
118 119 120
      }
    }
  } else {
121
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
122
    if (end >= 0) {
123
      forwardRows = end;
124

125
      if (pData[end + pos] == ekey) {
126
        forwardRows += 1;
127 128
      }
    }
X
Xiaoyu Wang 已提交
129 130 131 132 133 134 135 136
    //    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
    //    if (end >= 0) {
    //      forwardRows = pos - end;
    //
    //      if (pData[end] == ekey) {
    //        forwardRows += 1;
    //      }
    //    }
137 138
  }

139 140
  assert(forwardRows >= 0);
  return forwardRows;
141 142
}

5
54liuyao 已提交
143
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY*  keyList = (TSKEY*)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
160 161 162 163 164 165 166 167 168 169 170
      if (key >= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key < keyList[lastPos]) {
        lastPos += 1;
        if (lastPos >= num) {
          return -1;
        } else {
          return lastPos;
        }
      }
171 172 173 174 175 176

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        firstPos = midPos + 1;
177 178
      } else if (key > keyList[midPos]) {
        lastPos = midPos - 1;
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

X
Xiaoyu Wang 已提交
214 215
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
                                 __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num = -1;
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);

  if (order == TSDB_ORDER_ASC) {
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
      if (item != NULL) {
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
237
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
238 239
      }
    } else {
240
      num = pDataBlockInfo->rows - startPos;
241
      if (item != NULL) {
242
        item->lastKey = pDataBlockInfo->window.ekey + step;
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
      }
    }
  }

  assert(num >= 0);
  return num;
}

static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
    return;
  }

  int64_t key = tw->skey, interval = pInterval->interval;
  // convert key to second
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;

  if (pInterval->intervalUnit == 'y') {
    interval *= 12;
  }

  struct tm tm;
  time_t    t = (time_t)key;
  taosLocalTime(&t, &tm);

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
274
  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
275 276 277 278

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
wafwerar's avatar
wafwerar 已提交
279
  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
280 281 282 283

  tw->ekey -= 1;
}

284 285
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs,
                               int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
286
  SqlFunctionCtx* pCtx = pSup->pCtx;
287

288
  int32_t index = 1;
289
  for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
H
Haojun Liao 已提交
290
    if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
291 292 293 294
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

X
Xiaoyu Wang 已提交
295
    SFunctParam*     pParam = &pCtx[k].param[0];
296 297
    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId);

298
    ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey);
299

300
    double v1 = 0, v2 = 0, v = 0;
301
    if (prevRowIndex == -1) {
302
      SGroupKeys* p = taosArrayGet(pPrevValues, index);
303
      GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData);
304
    } else {
305
      GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex));
306 307
    }

308
    GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
309

310
#if 0
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            //            pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
330 331
#endif

X
Xiaoyu Wang 已提交
332 333 334
    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
    SPoint point = (SPoint){.key = windowKey, .val = &v};
335

X
Xiaoyu Wang 已提交
336
    taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
337

X
Xiaoyu Wang 已提交
338 339 340 341 342 343
    if (type == RESULT_ROW_START_INTERP) {
      pCtx[k].start.key = point.key;
      pCtx[k].start.val = v;
    } else {
      pCtx[k].end.key = point.key;
      pCtx[k].end.val = v;
344
    }
X
Xiaoyu Wang 已提交
345 346 347

    index += 1;
  }
348
#if 0
349
  }
350
#endif
351 352 353 354 355 356 357 358 359 360 361 362 363 364
}

static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

365 366
static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock,
                                              const TSKEY* tsCols, STimeWindow* win, SExprSupp* pSup) {
367
  bool ascQuery = (pInfo->inputOrder == TSDB_ORDER_ASC);
368

369
  TSKEY curTs = tsCols[pos];
370 371

  SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
X
Xiaoyu Wang 已提交
372
  TSKEY       lastTs = *(int64_t*)pTsKey->pData;
373 374 375 376 377

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery ? win->skey : win->ekey;
  if (key == curTs) {
378
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
379 380 381
    return true;
  }

382 383
  // it is the first time window, no need to do interpolation
  if (pTsKey->isNull && pos == 0) {
384
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
385 386
  } else {
    TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]);
387 388
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key,
                              RESULT_ROW_START_INTERP, pSup);
389 390 391 392 393
  }

  return true;
}

394 395 396
static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex,
                                            SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey,
                                            STimeWindow* win) {
397
  int32_t order = pInfo->inputOrder;
398 399

  TSKEY actualEndKey = tsCols[endRowIndex];
400
  TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey;
401 402

  // not ended in current data block, do not invoke interpolation
403
  if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) {
404
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
405 406 407
    return false;
  }

408
  // there is actual end point of current time window, no interpolation needs
409
  if (key == actualEndKey) {
410
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
411 412 413
    return true;
  }

414
  int32_t nextRowIndex = endRowIndex + 1;
415 416 417
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
418 419
  doTimeWindowInterpolation(pInfo->pPrevValues, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key,
                            RESULT_ROW_END_INTERP, pSup);
420 421 422
  return true;
}

5
54liuyao 已提交
423
bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo) {
424 425
  if (pInterval->interval != pInterval->sliding &&
      (pWin->ekey < pBlockInfo->calWin.skey || pWin->skey > pBlockInfo->calWin.ekey)) {
5
54liuyao 已提交
426 427 428 429 430
    return false;
  }
  return true;
}

431
static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
5
54liuyao 已提交
432
                                      TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
X
Xiaoyu Wang 已提交
433
  bool ascQuery = (order == TSDB_ORDER_ASC);
434 435 436 437 438 439 440 441 442 443

  int32_t precision = pInterval->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
    return -1;
  }

5
54liuyao 已提交
444 445 446 447
  if (!inSlidingWindow(pInterval, pNext, pDataBlockInfo) && order == TSDB_ORDER_ASC) {
    return -1;
  }

448
  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
449 450 451 452
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
453
    startPos = prevPosition + 1;
454
  } else {
455
    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
456 457
      startPos = 0;
    } else {
458
      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501
    }
  }

  /* interp query with fill should not skip time window */
  //  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
  //    return startPos;
  //  }

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
    if (ascQuery) {
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
      }
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
      }
    }
  }

  return startPos;
}

502 503
static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
  ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp == true;
  }
}

static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp = true;
  }
}

520 521
static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult,
                                        STimeWindow* win, int32_t startPos, int32_t forwardRows, SExprSupp* pSup) {
522
  if (!pInfo->timeWindowInterpo) {
523 524 525
    return;
  }

526
  ASSERT(pBlock != NULL);
527 528 529 530 531
  if (pBlock->pDataBlock == NULL) {
    //    tscError("pBlock->pDataBlock == NULL");
    return;
  }

532
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
533 534

  TSKEY* tsCols = (TSKEY*)(pColInfo->pData);
535
  bool   done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
536
  if (!done) {  // it is not interpolated, now start to generated the interpolated value
537
    bool interp = setTimeWindowInterpolationStartTs(pInfo, startPos, pBlock, tsCols, win, pSup);
538 539 540 541
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
542
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
543 544 545 546 547 548 549 550
  }

  // point interpolation does not require the end key time window interpolation.
  //  if (pointInterpQuery) {
  //    return;
  //  }

  // interpolation query does not generate the time window end interpolation
551
  done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
552
  if (!done) {
553
    int32_t endRowIndex = startPos + forwardRows - 1;
554

555
    TSKEY endKey = (pInfo->inputOrder == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
556
    bool  interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
557 558 559 560
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
561
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
562 563 564
  }
}

565 566
static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) {
  if (pBlock->pDataBlock == NULL) {
567 568 569
    return;
  }

570 571 572 573 574 575 576
  size_t num = taosArrayGetSize(pPrevKeys);
  for (int32_t k = 0; k < num; ++k) {
    SColumn* pc = taosArrayGet(pCols, k);

    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId);

    SGroupKeys* pkey = taosArrayGet(pPrevKeys, k);
X
Xiaoyu Wang 已提交
577
    for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) {
578 579 580 581 582 583 584 585 586 587 588 589 590 591
      if (colDataIsNull_s(pColInfo, i)) {
        continue;
      }

      char* val = colDataGetData(pColInfo, i);
      if (IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, varDataTLen(val));
        ASSERT(varDataTLen(val) <= pkey->bytes);
      } else {
        memcpy(pkey->pData, val, pkey->bytes);
      }

      break;
    }
592 593 594
  }
}

595 596 597 598
static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo,
                                       SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) {
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

599
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
600
  SExprSupp*                pSup = &pOperatorInfo->exprSupp;
601

602
  int32_t  startPos = 0;
603
  int32_t  numOfOutput = pSup->numOfExprs;
604
  uint64_t groupId = pBlock->info.groupId;
605

606
  SResultRow* pResult = NULL;
607

608 609
  while (1) {
    SListNode* pn = tdListGetHead(pResultRowInfo->openWindow);
610

611 612 613 614
    SResultRowPosition* p1 = (SResultRowPosition*)pn->data;
    if (p->pageId == p1->pageId && p->offset == p1->offset) {
      break;
    }
615

616
    SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1, false);
617
    ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
618

619
    if (pr->closed) {
X
Xiaoyu Wang 已提交
620 621
      ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
             isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
622 623 624
      tdListPopHead(pResultRowInfo->openWindow);
      continue;
    }
625

626
    STimeWindow w = pr->win;
627 628
    int32_t     ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx,
                                             numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
629 630 631 632 633 634
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP));

X
Xiaoyu Wang 已提交
635 636
    SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
    int64_t     prevTs = *(int64_t*)pTsKey->pData;
637 638
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey,
                              RESULT_ROW_END_INTERP, pSup);
639 640

    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
641
    setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
642

643
    doApplyFunctions(pTaskInfo, pSup->pCtx, &w, &pInfo->twAggSup.timeWindowData, startPos, 0, tsCols, pBlock->info.rows,
644
                     numOfExprs, pInfo->inputOrder);
645 646 647 648

    if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
      closeResultRow(pr);
      tdListPopHead(pResultRowInfo->openWindow);
X
Xiaoyu Wang 已提交
649
    } else {  // the remains are can not be closed yet.
650
      break;
651
    }
652
  }
653
}
654

5
54liuyao 已提交
655
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
5
54liuyao 已提交
656
  if (!pBlock || pBlock->info.rows == 0) {
5
54liuyao 已提交
657
    qDebug("===stream===printDataBlock: Block is Null or Empty");
5
54liuyao 已提交
658 659
    return;
  }
L
Liu Jicong 已提交
660
  char* pBuf = NULL;
5
54liuyao 已提交
661 662
  qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
  taosMemoryFree(pBuf);
5
54liuyao 已提交
663 664
}

5
54liuyao 已提交
665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720
typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);

int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;
      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;
      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;
      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

      if (comparefn(pKey, keyList, lastPos) > 0) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (comparefn(pKey, keyList, midPos) < 0) {
        lastPos = midPos - 1;
      } else if (comparefn(pKey, keyList, midPos) > 0) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

5
54liuyao 已提交
721
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
722

X
Xiaoyu Wang 已提交
723 724 725
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;
5
54liuyao 已提交
726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (key >= getValuefn(keyList, lastPos)) return lastPos;
      if (key == getValuefn(keyList, firstPos)) return firstPos;
      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (key <= getValuefn(keyList, firstPos)) return firstPos;
      if (key == getValuefn(keyList, lastPos)) return lastPos;

      if (key > getValuefn(keyList, lastPos)) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
772 773
  }

5
54liuyao 已提交
774 775 776
  return midPos;
}

5
54liuyao 已提交
777
int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {
L
Liu Jicong 已提交
778
  SArray*          res = (SArray*)data;
5
54liuyao 已提交
779
  SPullWindowInfo* pos = taosArrayGet(res, index);
L
Liu Jicong 已提交
780
  SPullWindowInfo* pData = (SPullWindowInfo*)pKey;
5
54liuyao 已提交
781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811
  if (pData->window.skey == pos->window.skey) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->window.skey > pos->window.skey) {
    return 1;
  }
  return -1;
}

static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {
  int32_t size = taosArrayGetSize(pPullWins);
  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);
  if (index == -1) {
    index = 0;
  } else {
    if (comparePullWinKey(pPullInfo, pPullWins, index) > 0) {
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
812 813 814
int32_t compareResKey(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
  SResKeyPos* pos = taosArrayGetP(res, index);
L
Liu Jicong 已提交
815
  SWinRes*    pData = (SWinRes*)pKey;
5
54liuyao 已提交
816 817 818 819 820 821 822 823 824 825 826 827 828
  if (pData->ts == *(int64_t*)pos->key) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (pData->ts > *(int64_t*)pos->key) {
    return 1;
  }
  return -1;
}

829
static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SArray* pUpdated) {
5
54liuyao 已提交
830
  int32_t size = taosArrayGetSize(pUpdated);
5
54liuyao 已提交
831 832
  SWinRes data = {.ts = ts, .groupId = groupId};
  int32_t index = binarySearchCom(pUpdated, size, &data, TSDB_ORDER_DESC, compareResKey);
5
54liuyao 已提交
833 834 835
  if (index == -1) {
    index = 0;
  } else {
5
54liuyao 已提交
836
    if (compareResKey(&data, pUpdated, index) > 0) {
5
54liuyao 已提交
837 838 839 840 841
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
H
Haojun Liao 已提交
842

5
54liuyao 已提交
843 844 845 846 847
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
5
54liuyao 已提交
848 849
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
X
Xiaoyu Wang 已提交
850
  if (taosArrayInsert(pUpdated, index, &newPos) == NULL) {
5
54liuyao 已提交
851 852 853 854 855
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
856 857 858 859 860 861 862 863 864 865 866 867 868
static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SHashObj* pUpdatedMap) {
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
  SWinRes key = {.ts = ts, .groupId = groupId};
  if (taosHashPut(pUpdatedMap, &key, sizeof(SWinRes), &newPos, sizeof(void*)) != TSDB_CODE_SUCCESS) {
    taosMemoryFree(newPos);
  }
  return TSDB_CODE_SUCCESS;
5
54liuyao 已提交
869 870
}

5
54liuyao 已提交
871 872 873 874 875 876
static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj* pUpdatedMap) {
  return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap);;
}

static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
  return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated);
5
54liuyao 已提交
877 878
}

5
54liuyao 已提交
879
static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) {
5
54liuyao 已提交
880 881
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
882
    SWinRes* pW = taosArrayGet(pWins, i);
5
54liuyao 已提交
883
    taosHashRemove(pUpdatedMap, pW, sizeof(SWinRes));
5
54liuyao 已提交
884 885 886
  }
}

887
int64_t getWinReskey(void* data, int32_t index) {
888
  SArray*  res = (SArray*)data;
889 890 891 892
  SWinRes* pos = taosArrayGet(res, index);
  return pos->ts;
}

5
54liuyao 已提交
893 894 895
int32_t compareWinRes(void* pKey, void* data, int32_t index) {
  SArray*     res = (SArray*)data;
  SWinRes*    pos = taosArrayGetP(res, index);
L
Liu Jicong 已提交
896
  SResKeyPos* pData = (SResKeyPos*)pKey;
5
54liuyao 已提交
897 898 899 900 901 902 903 904 905 906 907 908 909
  if (*(int64_t*)pData->key == pos->ts) {
    if (pData->groupId > pos->groupId) {
      return 1;
    } else if (pData->groupId < pos->groupId) {
      return -1;
    }
    return 0;
  } else if (*(int64_t*)pData->key > pos->ts) {
    return 1;
  }
  return -1;
}

5
54liuyao 已提交
910 911 912 913
static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) {
  if (!pUpdatedMap || taosHashGetSize(pUpdatedMap) == 0) {
    return;
  } 
914
  int32_t delSize = taosArrayGetSize(pDelWins);
5
54liuyao 已提交
915 916 917
  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    SResKeyPos* pResKey = (SResKeyPos*)pIte;
5
54liuyao 已提交
918 919
    int32_t     index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes);
    if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) {
920 921 922 923 924
      taosArrayRemove(pDelWins, index);
    }
  }
}

5
54liuyao 已提交
925 926 927 928 929
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) {
  ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
  return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark;
}

L
Liu Jicong 已提交
930
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); }
5
54liuyao 已提交
931

5
54liuyao 已提交
932
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
5
54liuyao 已提交
933
                            int32_t scanFlag, SHashObj* pUpdatedMap) {
934
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
935

936
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
937
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
938

X
Xiaoyu Wang 已提交
939
  int32_t     startPos = 0;
940
  int32_t     numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
941 942
  int64_t*    tsCols = extractTsCol(pBlock, pInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
943
  bool        ascScan = (pInfo->inputOrder == TSDB_ORDER_ASC);
X
Xiaoyu Wang 已提交
944 945
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;
946

947 948 949
  STimeWindow win =
      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder);
  int32_t ret = TSDB_CODE_SUCCESS;
950 951
  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
L
Liu Jicong 已提交
952 953
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
954 955 956
    if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
957

L
Liu Jicong 已提交
958
    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
959
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
D
dapan1121 已提交
960
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
5
54liuyao 已提交
961
    }
962 963
  }

X
Xiaoyu Wang 已提交
964 965
  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
966
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
967
  ASSERT(forwardRows > 0);
968 969

  // prev time window not interpolation yet.
970
  if (pInfo->timeWindowInterpo) {
971 972
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult);
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);
973 974

    // restore current time window
975 976
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
977 978 979 980
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

981
    // window start key interpolation
982
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
983
  }
984

985 986
  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
5
54liuyao 已提交
987 988
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
989
                     pBlock->info.rows, numOfOutput, pInfo->inputOrder);
5
54liuyao 已提交
990
  }
991 992

  doCloseWindow(pResultRowInfo, pInfo, pResult);
993 994 995

  STimeWindow nextWin = win;
  while (1) {
996
    int32_t prevEndPos = forwardRows - 1 + startPos;
997
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->inputOrder);
998 999 1000
    if (startPos < 0) {
      break;
    }
5
54liuyao 已提交
1001
    if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
5
54liuyao 已提交
1002 1003
      ekey = ascScan ? nextWin.ekey : nextWin.skey;
      forwardRows =
1004
          getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
5
54liuyao 已提交
1005 1006
      continue;
    }
1007 1008

    // null data, failed to allocate more memory buffer
X
Xiaoyu Wang 已提交
1009
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
1010
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1011 1012 1013 1014
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

L
Liu Jicong 已提交
1015
    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
1016
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
D
dapan1121 已提交
1017
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
1018 1019
    }

X
Xiaoyu Wang 已提交
1020
    ekey = ascScan ? nextWin.ekey : nextWin.skey;
1021
    forwardRows =
1022
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
1023 1024

    // window start(end) key interpolation
1025
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
1026 1027

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
1028
    doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
1029
                     pBlock->info.rows, numOfOutput, pInfo->inputOrder);
1030
    doCloseWindow(pResultRowInfo, pInfo, pResult);
1031 1032 1033
  }

  if (pInfo->timeWindowInterpo) {
1034
    saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
1035
  }
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047
}

void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
  // current result is done in computing final results.
  if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
    closeResultRow(pResult);
    tdListPopHead(pResultRowInfo->openWindow);
  }
}

SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult) {
  SResultRowPosition pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
X
Xiaoyu Wang 已提交
1048
  SListNode*         pn = tdListGetTail(pResultRowInfo->openWindow);
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
  if (pn == NULL) {
    tdListAppend(pResultRowInfo->openWindow, &pos);
    return pos;
  }

  SResultRowPosition* px = (SResultRowPosition*)pn->data;
  if (px->pageId != pos.pageId || px->offset != pos.offset) {
    tdListAppend(pResultRowInfo->openWindow, &pos);
  }

  return pos;
}

int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
  TSKEY* tsCols = NULL;
1064

1065 1066 1067 1068
  if (pBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;

1069 1070 1071 1072 1073 1074
    // no data in primary ts
    if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) {
      return NULL;
    }

    if (tsCols[0] != 0 && (pBlock->info.window.skey == 0 && pBlock->info.window.ekey == 0)) {
1075 1076 1077 1078 1079
      blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
    }
  }

  return tsCols;
1080 1081 1082 1083 1084 1085 1086
}

static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
  }

L
Liu Jicong 已提交
1087
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1088
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
1089
  SExprSupp*                pSup = &pOperator->exprSupp;
1090

1091 1092
  int32_t scanFlag = MAIN_SCAN;

X
Xiaoyu Wang 已提交
1093
  int64_t        st = taosGetTimestampUs();
1094 1095 1096
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1097
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1098 1099 1100 1101
    if (pBlock == NULL) {
      break;
    }

1102
    getTableScanInfo(pOperator, &pInfo->inputOrder, &scanFlag);
1103

1104
    if (pInfo->scalarSupp.pExprInfo != NULL) {
L
Liu Jicong 已提交
1105 1106
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
1107 1108
    }

1109
    // the pDataBlock are always the same one, no need to call this again
1110
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, scanFlag, true);
1111 1112
    blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);

H
Haojun Liao 已提交
1113
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL);
1114 1115
  }

1116
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->resultTsOrder);
1117
  OPTR_SET_OPENED(pOperator);
1118 1119

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1120 1121 1122
  return TSDB_CODE_SUCCESS;
}

1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
static bool compareVal(const char* v, const SStateKeys* pKey) {
  if (IS_VAR_DATA_TYPE(pKey->type)) {
    if (varDataLen(v) != varDataLen(pKey->pData)) {
      return false;
    } else {
      return strncmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0;
    }
  } else {
    return memcmp(pKey->pData, v, pKey->bytes) == 0;
  }
}

1135
static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
L
Liu Jicong 已提交
1136
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1137
  SExprSupp*     pSup = &pOperator->exprSupp;
1138

1139
  SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
1140 1141 1142
  int64_t          gid = pBlock->info.groupId;

  bool    masterScan = true;
1143
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1144 1145
  int16_t bytes = pStateColInfoData->info.bytes;

1146
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1147 1148 1149 1150 1151
  TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

1152
  struct SColumnDataAgg* pAgg = NULL;
1153
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
X
Xiaoyu Wang 已提交
1154
    pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
1155
    if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
1156 1157 1158 1159 1160
      continue;
    }

    char* val = colDataGetData(pStateColInfoData, j);

1161
    if (gid != pRowSup->groupId || !pInfo->hasKey) {
1162 1163 1164 1165 1166 1167 1168
      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }

1169 1170
      pInfo->hasKey = true;

1171 1172
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1173
    } else if (compareVal(val, &pInfo->stateKey)) {
1174
      doKeepTuple(pRowSup, tsList[j], gid);
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // a new state window started
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1185 1186
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1187 1188 1189 1190 1191
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
      }

      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1192
      doApplyFunctions(pTaskInfo, pSup->pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1193 1194 1195
                       pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);

      // here we start a new session window
1196 1197
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1198 1199 1200 1201 1202 1203 1204

      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }
1205 1206 1207 1208 1209
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
1210 1211
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1212 1213 1214 1215 1216
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
1217
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1218 1219 1220
                   pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
}

1221
static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
1222 1223 1224 1225 1226
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
1227

1228 1229
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
  SExprSupp*     pSup = &pOperator->exprSupp;
1230

1231
  SOptrBasicInfo* pBInfo = &pInfo->binfo;
1232 1233

  if (pOperator->status == OP_RES_TO_RETURN) {
1234
    while (1) {
1235
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1236
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1237

1238
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }

      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1249
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1250 1251
  }

1252
  int32_t order = TSDB_ORDER_ASC;
1253
  int64_t st = taosGetTimestampUs();
1254 1255 1256

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  while (1) {
1257
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1258 1259 1260 1261
    if (pBlock == NULL) {
      break;
    }

1262
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1263 1264
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1265 1266 1267
    doStateWindowAggImpl(pOperator, pInfo, pBlock);
  }

X
Xiaoyu Wang 已提交
1268
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1269 1270
  pOperator->status = OP_RES_TO_RETURN;

1271
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1272
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1273
  while (1) {
1274
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1275
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
1276

1277
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1278 1279 1280 1281
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
1282

1283 1284 1285 1286 1287
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
1288
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
1289 1290
}

1291
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
1292
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
1293
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1294 1295 1296 1297 1298 1299 1300 1301

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pBlock = pInfo->binfo.pRes;

  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
1302
    return pOperator->fpSet.getStreamResFn(pOperator);
1303 1304 1305 1306 1307 1308 1309
  } else {
    pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
    if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
      return NULL;
    }

    blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
1310 1311
    while (1) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1312
      doFilter(pInfo->pCondition, pBlock, NULL);
1313

1314
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
1315 1316 1317 1318
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
1319

1320 1321 1322
      if (pBlock->info.rows > 0) {
        break;
      }
1323 1324
    }

1325 1326 1327
    size_t rows = pBlock->info.rows;
    pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1328
    return (rows == 0) ? NULL : pBlock;
1329 1330 1331 1332
  }
}

// todo merged with the build group result.
1333
static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
1334
                                  int32_t* rowEntryInfoOffset) {
1335 1336 1337 1338 1339 1340 1341
  size_t num = taosArrayGetSize(pUpdateList);

  for (int32_t i = 0; i < num; ++i) {
    SResKeyPos* pPos = taosArrayGetP(pUpdateList, i);

    SFilePage*  bufPage = getBufPage(pBuf, pPos->pos.pageId);
    SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
1342

1343
    for (int32_t j = 0; j < numOfOutput; ++j) {
1344
      SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, j, rowEntryInfoOffset);
1345 1346
      if (pRow->numOfRows < pEntry->numOfRes) {
        pRow->numOfRows = pEntry->numOfRes;
1347 1348 1349 1350 1351 1352
      }
    }

    releaseBufPage(pBuf, bufPage);
  }
}
5
54liuyao 已提交
1353
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
L
Liu Jicong 已提交
1354
  for (int i = 0; i < num; i++) {
5
54liuyao 已提交
1355 1356
    if (type == STREAM_INVERT) {
      fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
L
Liu Jicong 已提交
1357
    } else if (type == STREAM_NORMAL) {
5
54liuyao 已提交
1358 1359 1360 1361
      fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
    }
  }
}
5
54liuyao 已提交
1362

1363
void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) {
1364
  SResultRow*     pResult = getResultRowByPos(pResultBuf, p1, false);
1365
  SqlFunctionCtx* pCtx = pSup->pCtx;
5
54liuyao 已提交
1366
  for (int32_t i = 0; i < numOfOutput; ++i) {
1367
    pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
1368 1369 1370 1371 1372 1373 1374 1375 1376
    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
      continue;
    }
    pResInfo->initialized = false;
    if (pCtx[i].functionId != -1) {
      pCtx[i].fpSet.init(&pCtx[i], pResInfo);
    }
  }
5
54liuyao 已提交
1377 1378 1379
  SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pResultBuf, bufPage);
5
54liuyao 已提交
1380 1381
}

5
54liuyao 已提交
1382
bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
X
Xiaoyu Wang 已提交
1383
                   int32_t numOfOutput) {
1384
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId);
5
54liuyao 已提交
1385
  SResultRowPosition* p1 =
1386
      (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1387 1388
  if (!p1) {
    // window has been closed
5
54liuyao 已提交
1389
    return false;
1390
  }
1391
  doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput);
5
54liuyao 已提交
1392
  return true;
5
54liuyao 已提交
1393 1394
}

1395 1396 1397 1398 1399 1400 1401 1402 1403
bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) {
  size_t bytes = sizeof(TSKEY);
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId);
  SResultRowPosition* p1 =
      (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
  if (!p1) {
    // window has been closed
    return false;
  }
5
54liuyao 已提交
1404
  // SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId);
1405 1406 1407 1408 1409 1410 1411
  // dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage);
  taosHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
  return true;
}

void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pUpWins, SInterval* pInterval) {
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
1412
  TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
1413
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
1414
  uint64_t*        groupIds = (uint64_t*)pGroupCol->pData;
1415 1416 1417
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
1418
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC);
1419 1420 1421 1422 1423 1424 1425 1426
    doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
    if (pUpWins) {
      SWinRes winRes = {.ts = win.skey, .groupId = groupIds[i]};
      taosArrayPush(pUpWins, &winRes);
    }
  }
}

1427 1428
static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput,
                           SSDataBlock* pBlock, SArray* pUpWins) {
1429 1430 1431 1432
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  TSKEY*           startTsCols = (TSKEY*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  TSKEY*           endTsCols = (TSKEY*)pEndTsCol->pData;
5
54liuyao 已提交
1433 1434
  uint64_t*        pGpDatas = NULL;
  if (pBlock->info.type == STREAM_RETRIEVE) {
5
54liuyao 已提交
1435
    SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
L
Liu Jicong 已提交
1436
    pGpDatas = (uint64_t*)pGpCol->pData;
5
54liuyao 已提交
1437
  }
L
Liu Jicong 已提交
1438 1439
  int32_t step = 0;
  int32_t startPos = 0;
1440 1441 1442 1443 1444 1445
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
    while (win.ekey <= endTsCols[i]) {
      uint64_t winGpId = pGpDatas ? pGpDatas[startPos] : pBlock->info.groupId;
L
Liu Jicong 已提交
1446
      bool     res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput);
1447 1448 1449 1450 1451
      if (pUpWins && res) {
        SWinRes winRes = {.ts = win.skey, .groupId = winGpId};
        taosArrayPush(pUpWins, &winRes);
      }
      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
5
54liuyao 已提交
1452
    }
5
54liuyao 已提交
1453 1454
  }
}
1455

5
54liuyao 已提交
1456
static int32_t getAllIntervalWindow(SHashObj* pHashMap, SHashObj* resWins) {
5
54liuyao 已提交
1457 1458 1459 1460 1461 1462
  void*  pIte = NULL;
  size_t keyLen = 0;
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    void*    key = taosHashGetKey(pIte, &keyLen);
    uint64_t groupId = *(uint64_t*)key;
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
1463
    TSKEY               ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1464
    SResultRowPosition* pPos = (SResultRowPosition*)pIte;
5
54liuyao 已提交
1465
    int32_t             code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
5
54liuyao 已提交
1466 1467 1468 1469 1470 1471 1472
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }
  return TSDB_CODE_SUCCESS;
}

1473
static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
5
54liuyao 已提交
1474
                                   SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pRecyPages,
1475
                                   SDiskbasedBuf* pDiscBuf) {
1476
  qDebug("===stream===close interval window");
X
Xiaoyu Wang 已提交
1477
  void*  pIte = NULL;
5
54liuyao 已提交
1478
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
1479 1480 1481
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    void*    key = taosHashGetKey(pIte, &keyLen);
    uint64_t groupId = *(uint64_t*)key;
5
54liuyao 已提交
1482
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
L
Liu Jicong 已提交
1483
    TSKEY       ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1484 1485 1486
    STimeWindow win;
    win.skey = ts;
    win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
L
Liu Jicong 已提交
1487 1488 1489
    SWinRes winRe = {
        .ts = win.skey,
        .groupId = groupId,
L
Liu Jicong 已提交
1490
    };
5
54liuyao 已提交
1491
    void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinRes));
5
54liuyao 已提交
1492
    if (isCloseWindow(&win, pSup)) {
5
54liuyao 已提交
1493
      if (chIds && pPullDataMap) {
L
Liu Jicong 已提交
1494
        SArray* chAy = *(SArray**)chIds;
5
54liuyao 已提交
1495
        int32_t size = taosArrayGetSize(chAy);
5
54liuyao 已提交
1496
        qDebug("===stream===window %" PRId64 " wait child size:%d", win.skey, size);
5
54liuyao 已提交
1497
        for (int32_t i = 0; i < size; i++) {
5
54liuyao 已提交
1498
          qDebug("===stream===window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i));
5
54liuyao 已提交
1499 1500 1501
        }
        continue;
      } else if (pPullDataMap) {
5
54liuyao 已提交
1502
        qDebug("===stream===close window %" PRId64, win.skey);
5
54liuyao 已提交
1503
      }
5
54liuyao 已提交
1504 1505
      SResultRowPosition* pPos = (SResultRowPosition*)pIte;
      if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
1506
        int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, closeWins);
5
54liuyao 已提交
1507 1508 1509
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
1510 1511 1512
        ASSERT(pRecyPages != NULL);
        taosArrayPush(pRecyPages, &pPos->pageId);
      } else {
5
54liuyao 已提交
1513
        // SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId);
1514
        // dBufSetBufPageRecycled(pDiscBuf, bufPage);
5
54liuyao 已提交
1515
      }
5
54liuyao 已提交
1516 1517 1518
      char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
      SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
      taosHashRemove(pHashMap, keyBuf, keyLen);
5
54liuyao 已提交
1519 1520 1521 1522 1523
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
1524 1525 1526
static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) {
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
L
Liu Jicong 已提交
1527
    SOperatorInfo*                    pChildOp = taosArrayGetP(pChildren, i);
5
54liuyao 已提交
1528 1529 1530
    SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
    ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
1531 1532
    closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, NULL,
                        pChInfo->aggSup.pResultBuf);
1533 1534 1535 1536 1537 1538 1539
  }
}

static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) {
  int32_t size = taosArrayGetSize(pageIds);
  for (int32_t i = 0; i < size; i++) {
    int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i);
5
54liuyao 已提交
1540
    // SFilePage* bufPage = getBufPage(pDiskBuf, pageId);
1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555
    // dBufSetBufPageRecycled(pDiskBuf, bufPage);
  }
  taosArrayClear(pageIds);
}

static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlock) {
  blockDataCleanup(pBlock);
  int32_t size = taosArrayGetSize(pWins);
  if (*index == size) {
    *index = 0;
    taosArrayClear(pWins);
    return;
  }
  blockDataEnsureCapacity(pBlock, size - *index);
  SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
1556
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
1557 1558 1559 1560 1561 1562
  for (int32_t i = *index; i < size; i++) {
    SWinRes* pWin = taosArrayGet(pWins, i);
    colDataAppend(pTsCol, pBlock->info.rows, (const char*)&pWin->ts, false);
    colDataAppend(pGroupCol, pBlock->info.rows, (const char*)&pWin->groupId, false);
    pBlock->info.rows++;
    (*index)++;
5
54liuyao 已提交
1563 1564 1565
  }
}

1566
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
1567
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
1568
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1569

1570
  pInfo->inputOrder = TSDB_ORDER_ASC;
1571
  SExprSupp* pSup = &pOperator->exprSupp;
1572 1573 1574 1575 1576 1577

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
1578
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
1579 1580 1581 1582
    if (pInfo->pDelRes->info.rows > 0) {
      return pInfo->pDelRes;
    }

1583
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1584
    if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
1585
      pOperator->status = OP_EXEC_DONE;
5
54liuyao 已提交
1586
      qDebug("===stream===single interval is done");
1587
      freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
1588
    }
5
54liuyao 已提交
1589
    printDataBlock(pInfo->binfo.pRes, "single interval");
1590 1591 1592 1593 1594
    return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];

1595
  SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);  // SResKeyPos
5
54liuyao 已提交
1596 1597
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP);
  SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
1598
  while (1) {
1599
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1600 1601 1602
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
1603
    // qInfo("===stream===%ld", pBlock->info.version);
1604
    printDataBlock(pBlock, "single interval recv");
1605

5
54liuyao 已提交
1606
    if (pBlock->info.type == STREAM_CLEAR) {
1607 1608
      doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock,
                     NULL);
1609
      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
5
54liuyao 已提交
1610
      continue;
1611 1612
    }
    if (pBlock->info.type == STREAM_DELETE_DATA) {
1613 1614
      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval);
      continue;
1615
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
1616
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
5
54liuyao 已提交
1617
      continue;
5
54liuyao 已提交
1618
    }
1619

1620
    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {
L
Liu Jicong 已提交
1621
      // set input version
1622 1623 1624
      pTaskInfo->version = pBlock->info.version;
    }

1625 1626 1627 1628 1629
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }

1630 1631 1632
    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
    // caller. Note that all the time window are not close till now.
    // the pDataBlock are always the same one, no need to call this again
1633
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, MAIN_SCAN, true);
1634
    if (pInfo->invertible) {
1635
      setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type);
1636 1637
    }

5
54liuyao 已提交
1638
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
1639
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap);
1640
  }
1641

1642
  pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
1643
  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap,
1644
                      pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
1645

5
54liuyao 已提交
1646 1647 1648 1649 1650 1651
  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosArraySort(pUpdated, resultrowComparAsc);

1652
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
1653 1654
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
1655
  removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
5
54liuyao 已提交
1656
  taosHashCleanup(pUpdatedMap);
1657 1658 1659 1660
  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows > 0) {
    return pInfo->pDelRes;
  }
1661

1662
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
1663
  printDataBlock(pInfo->binfo.pRes, "single interval");
1664 1665 1666 1667 1668
  return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}

static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
1669
  cleanupBasicInfo(&pInfo->binfo);
1670
  taosMemoryFreeClear(pInfo->stateKey.pData);
1671

D
dapan1121 已提交
1672
  taosMemoryFreeClear(param);
1673 1674
}

H
Haojun Liao 已提交
1675
static void freeItem(void* param) {
L
Liu Jicong 已提交
1676
  SGroupKeys* pKey = (SGroupKeys*)param;
H
Haojun Liao 已提交
1677 1678 1679
  taosMemoryFree(pKey->pData);
}

1680
void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) {
1681
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
1682
  cleanupBasicInfo(&pInfo->binfo);
1683
  cleanupAggSup(&pInfo->aggSup);
H
Haojun Liao 已提交
1684 1685 1686 1687 1688 1689 1690
  pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages);
  pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols);
  taosArrayDestroyEx(pInfo->pPrevValues, freeItem);

  pInfo->pPrevValues = NULL;
  pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins);
  pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);
1691

H
Haojun Liao 已提交
1692 1693
  cleanupGroupResInfo(&pInfo->groupResInfo);
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
D
dapan1121 已提交
1694
  taosMemoryFreeClear(param);
1695 1696
}

5
54liuyao 已提交
1697
void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
1698
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
1699
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
1700
  cleanupAggSup(&pInfo->aggSup);
L
Liu Jicong 已提交
1701
  // it should be empty.
5
54liuyao 已提交
1702 1703 1704
  taosHashCleanup(pInfo->pPullDataMap);
  taosArrayDestroy(pInfo->pPullWins);
  blockDataDestroy(pInfo->pPullDataRes);
1705
  taosArrayDestroy(pInfo->pRecycledPages);
H
Haojun Liao 已提交
1706
  blockDataDestroy(pInfo->pUpdateRes);
5
54liuyao 已提交
1707

1708 1709 1710 1711
  if (pInfo->pChildren) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
1712
      destroyStreamFinalIntervalOperatorInfo(pChildOp->info, numOfOutput);
1713 1714 1715
      taosMemoryFreeClear(pChildOp);
    }
  }
1716
  nodesDestroyNode((SNode*)pInfo->pPhyNode);
5
54liuyao 已提交
1717
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
1718

D
dapan1121 已提交
1719
  taosMemoryFreeClear(param);
5
54liuyao 已提交
1720 1721
}

1722
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
5
54liuyao 已提交
1723
  for (int32_t i = 0; i < numOfCols; i++) {
5
54liuyao 已提交
1724
    if (fmIsUserDefinedFunc(pFCtx[i].functionId) || !fmIsInvertible(pFCtx[i].functionId)) {
5
54liuyao 已提交
1725 1726 1727 1728 1729 1730
      return false;
    }
  }
  return true;
}

1731
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
1732 1733
  // the primary timestamp column
  bool needed = false;
1734 1735
  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
1736

X
Xiaoyu Wang 已提交
1737
  {  // ts column
1738 1739
    SColumn c = {0};
    c.colId = 1;
1740
    c.slotId = pInfo->primaryTsIndex;
1741 1742
    c.type = TSDB_DATA_TYPE_TIMESTAMP;
    c.bytes = sizeof(int64_t);
1743
    taosArrayPush(pInfo->pInterpCols, &c);
1744 1745

    SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1746 1747 1748 1749
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = true;  // to denote no value is assigned yet
    key.pData = taosMemoryCalloc(1, c.bytes);
1750
    taosArrayPush(pInfo->pPrevValues, &key);
1751 1752
  }

X
Xiaoyu Wang 已提交
1753
  for (int32_t i = 0; i < numOfCols; ++i) {
1754 1755
    SExprInfo* pExpr = pCtx[i].pExpr;

H
Haojun Liao 已提交
1756
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1757 1758 1759
      SFunctParam* pParam = &pExpr->base.pParam[0];

      SColumn c = *pParam->pCol;
1760
      taosArrayPush(pInfo->pInterpCols, &c);
1761 1762 1763
      needed = true;

      SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1764 1765
      key.bytes = c.bytes;
      key.type = c.type;
1766
      key.isNull = false;
X
Xiaoyu Wang 已提交
1767
      key.pData = taosMemoryCalloc(1, c.bytes);
1768
      taosArrayPush(pInfo->pPrevValues, &key);
1769 1770 1771 1772 1773 1774
    }
  }

  return needed;
}

1775
void increaseTs(SqlFunctionCtx* pCtx) {
1776
  if (pCtx[0].pExpr->pExpr->_function.pFunctNode->funcType == FUNCTION_TYPE_WSTART) {
1777 1778 1779 1780
    pCtx[0].increase = true;
  }
}

1781
void initIntervalDownStream(SOperatorInfo* downstream, uint8_t type, SAggSupporter* pSup) {
1782 1783 1784 1785
  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
    // Todo(liuyao) support partition by column
    return;
  }
5
54liuyao 已提交
1786 1787
  SStreamScanInfo* pScanInfo = downstream->info;
  pScanInfo->sessionSup.parentType = type;
1788
  pScanInfo->sessionSup.pIntervalAggSup = pSup;
5
54liuyao 已提交
1789 1790
}

1791 1792
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                          SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
L
Liu Jicong 已提交
1793 1794
                                          STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode,
                                          SExecTaskInfo* pTaskInfo, bool isStream) {
1795
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1796
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1797 1798 1799 1800
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1801
  pOperator->pTaskInfo = pTaskInfo;
L
Liu Jicong 已提交
1802
  pInfo->win = pTaskInfo->window;
1803 1804
  pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
  pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
L
Liu Jicong 已提交
1805
  pInfo->interval = *pInterval;
L
Liu Jicong 已提交
1806
  pInfo->execModel = pTaskInfo->execModel;
L
Liu Jicong 已提交
1807
  pInfo->twAggSup = *pTwAggSupp;
5
54liuyao 已提交
1808
  pInfo->ignoreExpiredData = pPhyNode->window.igExpired;
1809
  pInfo->pCondition = pPhyNode->window.node.pConditions;
1810
  pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock;
1811 1812 1813 1814

  if (pPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
L
Liu Jicong 已提交
1815
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
1816 1817 1818 1819
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
1820

1821
  pInfo->primaryTsIndex = primaryTsSlotId;
1822 1823
  SExprSupp* pSup = &pOperator->exprSupp;

1824
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
1825
  initResultSizeInfo(&pOperator->resultInfo, 4096);
1826

1827 1828 1829
  int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);

1830 1831
  if (isStream) {
    ASSERT(numOfCols > 0);
1832
    increaseTs(pSup->pCtx);
1833
  }
1834

1835
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
1836

1837
  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
X
Xiaoyu Wang 已提交
1838
  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
1839

1840
  pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, pInfo);
1841 1842
  if (pInfo->timeWindowInterpo) {
    pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
H
Haojun Liao 已提交
1843 1844 1845
    if (pInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
1846
  }
1847 1848 1849
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes));
  pInfo->delIndex = 0;
1850
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
1851
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1852

X
Xiaoyu Wang 已提交
1853 1854 1855 1856
  pOperator->name = "TimeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
1857 1858
  pOperator->exprSupp.pExprInfo = pExprInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
X
Xiaoyu Wang 已提交
1859
  pOperator->info = pInfo;
1860 1861 1862 1863

  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL,
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

5
54liuyao 已提交
1864
  if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) {
1865
    initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup);
5
54liuyao 已提交
1866 1867
  }

1868 1869 1870 1871 1872 1873 1874
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1875
_error:
1876 1877 1878 1879 1880 1881 1882 1883 1884
  destroyIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                                SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
wmmhello's avatar
wmmhello 已提交
1885
                                                STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
1886
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1887
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1888 1889 1890 1891
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1892
  pOperator->pTaskInfo = pTaskInfo;
1893
  pInfo->inputOrder = TSDB_ORDER_ASC;
1894 1895 1896 1897 1898 1899 1900 1901 1902
  pInfo->interval = *pInterval;
  pInfo->execModel = OPTR_EXEC_MODEL_STREAM;
  pInfo->win = pTaskInfo->window;
  pInfo->twAggSup = *pTwAggSupp;
  pInfo->primaryTsIndex = primaryTsSlotId;

  int32_t numOfRows = 4096;
  size_t  keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

1903
  initResultSizeInfo(&pOperator->resultInfo, numOfRows);
1904 1905
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);
1906 1907
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);

wmmhello's avatar
wmmhello 已提交
1908
  if (code != TSDB_CODE_SUCCESS) {
1909 1910 1911
    goto _error;
  }

1912
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1913 1914

  pOperator->name = "StreamTimeIntervalAggOperator";
X
Xiaoyu Wang 已提交
1915
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
1916
  pOperator->blocking = true;
1917
  pOperator->status = OP_NOT_OPENED;
1918 1919
  pOperator->exprSupp.pExprInfo = pExprInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931
  pOperator->info = pInfo;

  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL,
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1932
_error:
1933 1934 1935 1936 1937 1938 1939
  destroyIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

1940
// todo handle multiple timeline cases. assume no timeline interweaving
1941 1942
static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1943
  SExprSupp*     pSup = &pOperator->exprSupp;
1944

1945
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1946 1947

  bool    masterScan = true;
1948
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963
  int64_t gid = pBlock->info.groupId;

  int64_t gap = pInfo->gap;

  if (!pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->winSup.prevTs = INT64_MIN;
  }

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

  // In case of ascending or descending order scan data, only one time window needs to be kepted for each table.
  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
1964 1965 1966 1967 1968
    if (gid != pRowSup->groupId || pInfo->winSup.prevTs == INT64_MIN) {
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
    } else if ((tsList[j] - pRowSup->prevTs >= 0) && tsList[j] - pRowSup->prevTs <= gap ||
            (pRowSup->prevTs - tsList[j] >= 0 ) && (pRowSup->prevTs - tsList[j] <= gap)) {
1969
      // The gap is less than the threshold, so it belongs to current session window that has been opened already.
1970
      doKeepTuple(pRowSup, tsList[j], gid);
1971 1972 1973 1974 1975 1976 1977 1978 1979 1980
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
1981 1982
      int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1983 1984 1985 1986 1987 1988
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
      }

      // pInfo->numOfRows data belong to the current session window
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1989
      doApplyFunctions(pTaskInfo, pSup->pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1990 1991 1992
                       pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);

      // here we start a new session window
1993 1994
      doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
      doKeepTuple(pRowSup, tsList[j], gid);
1995 1996 1997 1998 1999
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
2000 2001
  int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
2002 2003 2004 2005 2006
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
2007
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
2008 2009 2010
                   pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
}

2011
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
2012 2013 2014 2015 2016 2017
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*          pBInfo = &pInfo->binfo;
2018
  SExprSupp*               pSup = &pOperator->exprSupp;
2019 2020

  if (pOperator->status == OP_RES_TO_RETURN) {
2021
    while (1) {
2022
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
2023
      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
2024

2025
      bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
2026 2027 2028 2029
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
2030

2031 2032 2033 2034 2035
      if (pBInfo->pRes->info.rows > 0) {
        break;
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
2036
    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
2037 2038
  }

2039 2040 2041
  int64_t st = taosGetTimestampUs();
  int32_t order = TSDB_ORDER_ASC;

2042 2043 2044
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
2045
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
2046 2047 2048 2049 2050
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
2051
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
2052 2053
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

2054 2055 2056
    doSessionWindowAggImpl(pOperator, pInfo, pBlock);
  }

2057 2058
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;

2059 2060 2061
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;

2062
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
2063
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
2064
  while (1) {
2065
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
2066
    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
2067

2068
    bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
2069 2070 2071 2072
    if (!hasRemain) {
      doSetOperatorCompleted(pOperator);
      break;
    }
2073

2074 2075 2076 2077 2078
    if (pBInfo->pRes->info.rows > 0) {
      break;
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
2079
  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
2080 2081
}

2082
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
H
Haojun Liao 已提交
2083
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2084 2085
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
H
Haojun Liao 已提交
2086 2087 2088 2089 2090 2091

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);

      pkey->isNull = false;
2092
      char* val = colDataGetData(pColInfoData, rowIndex);
H
Haojun Liao 已提交
2093 2094 2095
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }
2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115

  pSliceInfo->isPrevRowSet = true;
}

static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i);

      pkey->isNull = false;
      char* val = colDataGetData(pColInfoData, rowIndex);
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }

  pSliceInfo->isNextRowSet = true;
H
Haojun Liao 已提交
2116 2117
}

2118 2119
static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex,
                             bool isLastRow) {
2120
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2121
  bool fillLastPoint = pSliceInfo->fillLastPoint;
2122 2123
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
2124 2125
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
    SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i);
2126 2127 2128

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
      if (isLastRow) {
        pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
      } else if (fillLastPoint) {
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
      } else {
        pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
        pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1);

        char* val;
        val = colDataGetData(pColInfoData, rowIndex);
        memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes);
        val = colDataGetData(pColInfoData, rowIndex + 1);
        memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes);
      }
2145 2146 2147 2148

      pLinearInfo->hasNull = false;
    } else {
      pLinearInfo->hasNull = true;
2149 2150 2151
    }
  }

2152 2153
  pSliceInfo->fillLastPoint = isLastRow ? true : false;

2154 2155
}

2156
static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup,
2157
                                   SSDataBlock* pResBlock) {
2158 2159 2160 2161 2162 2163 2164 2165
  int32_t rows = pResBlock->info.rows;

  // todo set the correct primary timestamp column

  // output the result
  for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
    SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
    int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
2166
    int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
2167

2168
    //SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
2169 2170 2171
    SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

    switch (pSliceInfo->fillType) {
2172
      case TSDB_FILL_NULL: {
2173
        colDataAppendNULL(pDst, rows);
2174
        pResBlock->info.rows += 1;
2175
        break;
2176
      }
2177 2178 2179 2180 2181 2182 2183

      case TSDB_FILL_SET_VALUE: {
        SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal;

        if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
          float v = 0;
          GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
2184
          colDataAppend(pDst, rows, (char *)&v, false);
2185 2186 2187
        } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
          double v = 0;
          GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
2188
          colDataAppend(pDst, rows, (char *)&v, false);
2189 2190 2191
        } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
          int64_t v = 0;
          GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
2192
          colDataAppend(pDst, rows, (char *)&v, false);
2193
        }
2194 2195 2196
        pResBlock->info.rows += 1;
        break;
      }
2197

2198
      case TSDB_FILL_LINEAR: {
2199
        SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, srcSlot);
2200

2201 2202 2203 2204 2205
        SPoint start   = pLinearInfo->start;
        SPoint end     = pLinearInfo->end;
        SPoint current = {.key = pSliceInfo->current};
        current.val = taosMemoryCalloc(pLinearInfo->bytes, 1);

2206 2207 2208 2209 2210
        // before interp range, do not fill
        if (start.key == INT64_MIN || end.key == INT64_MAX) {
          break;
        }

2211 2212 2213 2214 2215 2216 2217 2218
        if (pLinearInfo->hasNull) {
          colDataAppendNULL(pDst, rows);
        } else {
          taosGetLinearInterpolationVal(&current, pLinearInfo->type, &start, &end, pLinearInfo->type);
          colDataAppend(pDst, rows, (char *)current.val, false);
        }

        pResBlock->info.rows += 1;
2219
        break;
2220
      }
2221
      case TSDB_FILL_PREV: {
2222
        if (!pSliceInfo->isPrevRowSet) {
2223 2224 2225
          break;
        }

2226 2227
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
        colDataAppend(pDst, rows, pkey->pData, false);
2228 2229 2230
        pResBlock->info.rows += 1;
        break;
      }
2231 2232

      case TSDB_FILL_NEXT: {
2233
        if (!pSliceInfo->isNextRowSet) {
2234 2235 2236
          break;
        }

2237 2238
        SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot);
        colDataAppend(pDst, rows, pkey->pData, false);
2239 2240 2241
        pResBlock->info.rows += 1;
        break;
      }
2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259

      case TSDB_FILL_NONE:
      default:
        break;
    }
  }
}

static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pPrevRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pPrevRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

2260
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pPrevRow, &key);
  }

2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300
  pInfo->isPrevRowSet = false;

  return TSDB_CODE_SUCCESS;
}

static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pNextRow != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys));
  if (pInfo->pNextRow == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

    SGroupKeys key = {0};
    key.bytes = pColInfo->info.bytes;
    key.type = pColInfo->info.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
    taosArrayPush(pInfo->pNextRow, &key);
  }

  pInfo->isNextRowSet = false;

2301 2302 2303
  return TSDB_CODE_SUCCESS;
}

2304 2305 2306 2307 2308 2309
static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  if (pInfo->pLinearInfo != NULL) {
    return TSDB_CODE_SUCCESS;
  }

  pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo));
2310
  if (pInfo->pLinearInfo == NULL) {
2311 2312 2313 2314 2315 2316 2317
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);

2318 2319 2320 2321 2322 2323 2324 2325 2326
    SFillLinearInfo linearInfo = {0};
    linearInfo.start.key = INT64_MIN;
    linearInfo.end.key   = INT64_MAX;
    linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes);
    linearInfo.end.val   = taosMemoryCalloc(1, pColInfo->info.bytes);
    linearInfo.hasNull   = false;
    linearInfo.type  = pColInfo->info.type;
    linearInfo.bytes = pColInfo->info.bytes;
    taosArrayPush(pInfo->pLinearInfo, &linearInfo);
2327 2328
  }

2329 2330
  pInfo->fillLastPoint = false;

2331 2332 2333
  return TSDB_CODE_SUCCESS;
}

2334 2335 2336 2337
static bool needToFillLastPoint(STimeSliceOperatorInfo* pSliceInfo) {
  return (pSliceInfo->fillLastPoint == true && pSliceInfo->fillType == TSDB_FILL_LINEAR);
}

2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357
static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
  int32_t code;
  code = initPrevRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initNextRowsKeeper(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  code = initFillLinearInfo(pInfo, pBlock);
  if (code != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_FAILED;
  }

  return TSDB_CODE_SUCCESS;
}

H
Haojun Liao 已提交
2358
static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
2359 2360 2361 2362
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

2363 2364
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

2365
  STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
2366 2367
  SSDataBlock*            pResBlock = pSliceInfo->pRes;
  SExprSupp*              pSup = &pOperator->exprSupp;
H
Haojun Liao 已提交
2368

2369 2370
  //  if (pOperator->status == OP_RES_TO_RETURN) {
  //    //    doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes);
2371
  //    if (pResBlock->info.rows == 0 || !hasRemainResults(&pSliceInfo->groupResInfo)) {
2372 2373 2374 2375 2376
  //      doSetOperatorCompleted(pOperator);
  //    }
  //
  //    return pResBlock;
  //  }
2377

2378 2379
  int32_t        order = TSDB_ORDER_ASC;
  SInterval*     pInterval = &pSliceInfo->interval;
2380 2381
  SOperatorInfo* downstream = pOperator->pDownstream[0];

2382 2383
  blockDataCleanup(pResBlock);

2384
  while (1) {
2385
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
2386 2387 2388 2389
    if (pBlock == NULL) {
      break;
    }

2390
    int32_t code = initKeeperInfo(pSliceInfo, pBlock);
2391 2392 2393 2394
    if (code != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, code);
    }

2395
    // the pDataBlock are always the same one, no need to call this again
2396
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
H
Haojun Liao 已提交
2397

2398
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
2399 2400
    for (int32_t i = 0; i < pBlock->info.rows; ++i) {
      int64_t ts = *(int64_t*)colDataGetData(pTsCol, i);
H
Haojun Liao 已提交
2401

2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418
      if (i == 0 && needToFillLastPoint(pSliceInfo)) { // first row in current block
        doKeepLinearInfo(pSliceInfo, pBlock, i, false);
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
          genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
        }

        if (pSliceInfo->current > pSliceInfo->win.ekey) {
          doSetOperatorCompleted(pOperator);
          break;
        }
      }

H
Haojun Liao 已提交
2419
      if (ts == pSliceInfo->current) {
2420
        for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
2421
          SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
2422 2423
          int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
          int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
H
Haojun Liao 已提交
2424 2425

          SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
2426
          SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
H
Haojun Liao 已提交
2427 2428

          char* v = colDataGetData(pSrc, i);
2429
          colDataAppend(pDst, pResBlock->info.rows, v, false);
H
Haojun Liao 已提交
2430 2431
        }

2432
        pResBlock->info.rows += 1;
2433
        doKeepPrevRows(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2434

2435 2436
        // for linear interpolation, always fill value between this and next points;
        // if its the first point in data block, also fill values between previous(if there's any) and this point;
2437 2438
        // if its the last point in data block, no need to fill, but reserve this point as the start value and do
        // the interpolation when processing next data block.
2439
        if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2440
          doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2441 2442
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2443 2444 2445 2446
          if (i < pBlock->info.rows - 1) {
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2447
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459
                pSliceInfo->current =
                    taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }

              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
            }
2460 2461 2462
          } else {// it is the last row of current block
            //store ts value as start, and calculate interp value when processing next block
            doKeepLinearInfo(pSliceInfo, pBlock, i, true);
2463
          }
2464 2465 2466 2467 2468 2469 2470
        } else { // non-linear interpolation
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (pSliceInfo->current > pSliceInfo->win.ekey) {
            doSetOperatorCompleted(pOperator);
            break;
          }
2471

2472 2473 2474
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
2475
        }
H
Haojun Liao 已提交
2476
      } else if (ts < pSliceInfo->current) {
2477
        // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate
2478 2479
        doKeepPrevRows(pSliceInfo, pBlock, i);

2480
        if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2481
          doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2482
          // no need to increate pSliceInfo->current here
2483 2484 2485 2486 2487 2488
          //pSliceInfo->current =
          //    taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
          if (i < pBlock->info.rows - 1) {
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2489
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
2490 2491 2492 2493 2494 2495 2496 2497 2498
                pSliceInfo->current =
                    taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }

              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
H
Haojun Liao 已提交
2499 2500
                break;
              }
H
Haojun Liao 已提交
2501
            }
2502 2503 2504 2505 2506 2507 2508 2509
          }
        } else { // non-linear interpolation
          if (i < pBlock->info.rows - 1) {
            // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
            doKeepNextRows(pSliceInfo, pBlock, i + 1);
            int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
            if (nextTs > pSliceInfo->current) {
              while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2510
                genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
2511 2512 2513 2514 2515 2516
                pSliceInfo->current =
                    taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
                if (pResBlock->info.rows >= pResBlock->info.capacity) {
                  break;
                }
              }
2517

2518 2519 2520 2521 2522 2523
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
            } else {
              // ignore current row, and do nothing
H
Haojun Liao 已提交
2524
            }
2525 2526
          } else {  // it is the last row of current block
            doKeepPrevRows(pSliceInfo, pBlock, i);
H
Haojun Liao 已提交
2527
          }
2528 2529
        }
      } else {  // ts > pSliceInfo->current
2530
        // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
2531 2532
        doKeepNextRows(pSliceInfo, pBlock, i);

2533
        while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
2534
          genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
2535 2536
          pSliceInfo->current =
              taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
2537 2538 2539
          if (pResBlock->info.rows >= pResBlock->info.capacity) {
            break;
          }
2540 2541
        }

2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559
        // add current row if timestamp match
        if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
          for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
            SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
            int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
            int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;

            SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
            SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);

            char* v = colDataGetData(pSrc, i);
            colDataAppend(pDst, pResBlock->info.rows, v, false);
          }

          pResBlock->info.rows += 1;
          doKeepPrevRows(pSliceInfo, pBlock, i);


2560
          if (pSliceInfo->fillType == TSDB_FILL_LINEAR) {
2561
            doKeepLinearInfo(pSliceInfo, pBlock, i, false);
2562 2563 2564 2565 2566 2567
            pSliceInfo->current =
                taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
            if (i < pBlock->info.rows - 1) {
              int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
              if (nextTs > pSliceInfo->current) {
                while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
2568
                  genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588
                  pSliceInfo->current =
                      taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
                  if (pResBlock->info.rows >= pResBlock->info.capacity) {
                    break;
                  }
                }

                if (pSliceInfo->current > pSliceInfo->win.ekey) {
                  doSetOperatorCompleted(pOperator);
                  break;
                }
              }
            }
          } else { // non-linear interpolation
            pSliceInfo->current =
                taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);

            if (pResBlock->info.rows >= pResBlock->info.capacity) {
              break;
            }
2589 2590 2591
          }
        }

2592 2593 2594
        if (pSliceInfo->current > pSliceInfo->win.ekey) {
          doSetOperatorCompleted(pOperator);
          break;
H
Haojun Liao 已提交
2595 2596 2597
        }
      }
    }
2598

2599 2600 2601 2602 2603 2604 2605 2606 2607 2608
  }

  // check if need to interpolate after last datablock
  // except for fill(next), fill(linear)
  while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) {
    genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock);
    pSliceInfo->current =
        taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
    if (pResBlock->info.rows >= pResBlock->info.capacity) {
      break;
2609
    }
2610 2611 2612 2613
  }

  // restore the value
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
2614
  if (pResBlock->info.rows == 0) {
2615 2616 2617
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
2618 2619 2620
  return pResBlock->info.rows == 0 ? NULL : pResBlock;
}

2621
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
2622 2623 2624 2625 2626 2627
  STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pOperator == NULL || pInfo == NULL) {
    goto _error;
  }

2628
  SInterpFuncPhysiNode* pInterpPhyNode = (SInterpFuncPhysiNode*)pPhyNode;
2629
  SExprSupp*            pSup = &pOperator->exprSupp;
2630

2631
  int32_t    numOfExprs = 0;
2632
  SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
2633
  int32_t    code = initExprSupp(pSup, pExprInfo, numOfExprs);
H
Haojun Liao 已提交
2634 2635 2636 2637
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2638
  if (pInterpPhyNode->pExprs != NULL) {
2639
    int32_t    num = 0;
2640 2641 2642 2643 2644 2645 2646 2647 2648
    SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

  pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries);
  pInfo->fillType = convertFillType(pInterpPhyNode->fillMode);
2649
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2650

2651 2652 2653
  pInfo->pPrevRow = NULL;
  pInfo->pNextRow = NULL;
  pInfo->pLinearInfo = NULL;
2654
  pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, (SNodeListNode*)pInterpPhyNode->pFillValues);
L
Liu Jicong 已提交
2655 2656
  pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  pInfo->win = pInterpPhyNode->timeRange;
2657
  pInfo->interval.interval = pInterpPhyNode->interval;
L
Liu Jicong 已提交
2658
  pInfo->current = pInfo->win.skey;
H
Haojun Liao 已提交
2659

2660
  pOperator->name = "TimeSliceOperator";
2661
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC;
2662 2663 2664 2665
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;
  pOperator->pTaskInfo = pTaskInfo;
2666

2667 2668
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyBasicOperatorInfo, NULL, NULL, NULL);
2669

2670 2671
  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);

H
Haojun Liao 已提交
2672
  code = appendDownstream(pOperator, &downstream, 1);
2673 2674
  return pOperator;

L
Liu Jicong 已提交
2675
_error:
2676 2677 2678 2679 2680 2681 2682
  taosMemoryFree(pInfo);
  taosMemoryFree(pOperator);
  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
  return NULL;
}

SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
2683
                                             SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId,
2684
                                             SColumn* pStateKeyCol, SNode* pCondition, SExecTaskInfo* pTaskInfo) {
2685 2686 2687 2688 2689 2690
  SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

2691 2692 2693 2694
  pInfo->stateCol = *pStateKeyCol;
  pInfo->stateKey.type = pInfo->stateCol.type;
  pInfo->stateKey.bytes = pInfo->stateCol.bytes;
  pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
2695
  pInfo->pCondition = pCondition;
2696 2697 2698 2699
  if (pInfo->stateKey.pData == NULL) {
    goto _error;
  }

2700 2701
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

2702
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2703 2704 2705
  initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExpr, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);

2706
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2707

L
Liu Jicong 已提交
2708
  pInfo->twAggSup = *pTwAggSup;
2709 2710
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

X
Xiaoyu Wang 已提交
2711 2712
  pInfo->tsSlotId = tsSlotId;
  pOperator->name = "StateWindowOperator";
2713
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE;
X
Xiaoyu Wang 已提交
2714 2715
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
2716 2717
  pOperator->exprSupp.pExprInfo = pExpr;
  pOperator->exprSupp.numOfExprs = numOfCols;
X
Xiaoyu Wang 已提交
2718 2719
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
2720 2721 2722 2723 2724 2725 2726

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL,
                                         destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  int32_t code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

L
Liu Jicong 已提交
2727
_error:
2728 2729 2730 2731 2732 2733
  pTaskInfo->code = TSDB_CODE_SUCCESS;
  return NULL;
}

void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
2734
  cleanupBasicInfo(&pInfo->binfo);
2735

H
Haojun Liao 已提交
2736 2737 2738 2739
  colDataDestroy(&pInfo->twAggSup.timeWindowData);

  cleanupAggSup(&pInfo->aggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
D
dapan1121 已提交
2740
  taosMemoryFreeClear(param);
2741 2742
}

H
Haojun Liao 已提交
2743
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode,
2744
                                            SExecTaskInfo* pTaskInfo) {
2745 2746 2747 2748 2749 2750
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

2751
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
2752
  initResultSizeInfo(&pOperator->resultInfo, 4096);
2753

2754
  int32_t      numOfCols = 0;
H
Haojun Liao 已提交
2755 2756 2757
  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc);

2758
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
2759 2760 2761 2762
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2763 2764
  initBasicInfo(&pInfo->binfo, pResBlock);

H
Haojun Liao 已提交
2765 2766 2767 2768
  pInfo->twAggSup.waterMark = pSessionNode->window.watermark;
  pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType;
  pInfo->gap = pSessionNode->gap;

2769
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2770 2771
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

2772 2773
  pInfo->tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  pInfo->binfo.pRes = pResBlock;
L
Liu Jicong 已提交
2774
  pInfo->winSup.prevTs = INT64_MIN;
2775 2776
  pInfo->reptScan = false;
  pInfo->pCondition = pSessionNode->window.node.pConditions;
H
Haojun Liao 已提交
2777

L
Liu Jicong 已提交
2778
  pOperator->name = "SessionWindowAggOperator";
2779
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
2780
  pOperator->blocking = true;
L
Liu Jicong 已提交
2781
  pOperator->status = OP_NOT_OPENED;
2782 2783
  pOperator->exprSupp.pExprInfo = pExprInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
L
Liu Jicong 已提交
2784
  pOperator->info = pInfo;
2785 2786 2787 2788 2789 2790 2791 2792

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL,
                                         destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  pOperator->pTaskInfo = pTaskInfo;

  code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

L
Liu Jicong 已提交
2793
_error:
2794 2795 2796 2797 2798 2799 2800 2801
  if (pInfo != NULL) {
    destroySWindowOperatorInfo(pInfo, numOfCols);
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
2802
}
5
54liuyao 已提交
2803

5
54liuyao 已提交
2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821
void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
                      SExecTaskInfo* pTaskInfo) {
  for (int32_t k = 0; k < numOfOutput; ++k) {
    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
      continue;
    }
    int32_t code = TSDB_CODE_SUCCESS;
    if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
      code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
      if (code != TSDB_CODE_SUCCESS) {
        qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
        pTaskInfo->code = code;
        longjmp(pTaskInfo->env, code);
      }
    }
  }
}

2822 2823 2824 2825 2826 2827 2828 2829
bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) {
  int32_t bytes = sizeof(TSKEY);
  SET_RES_WINDOW_KEY(pSup->keyBuf, &ts, bytes, groupId);
  SResultRowPosition* p1 =
      (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
  return p1 != NULL;
}

2830 2831
static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray,
                                  int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SArray* pUpdated) {
5
54liuyao 已提交
2832
  int32_t size = taosArrayGetSize(pWinArray);
5
54liuyao 已提交
2833 2834 2835
  if (!pInfo->pChildren) {
    return;
  }
5
54liuyao 已提交
2836
  for (int32_t i = 0; i < size; i++) {
2837 2838 2839 2840 2841
    SWinRes*    pWinRes = taosArrayGet(pWinArray, i);
    SResultRow* pCurResult = NULL;
    STimeWindow ParentWin = {.skey = pWinRes->ts, .ekey = pWinRes->ts + 1};
    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &ParentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx,
                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2842
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
2843
    bool    find = true;
5
54liuyao 已提交
2844 2845 2846
    for (int32_t j = 0; j < numOfChildren; j++) {
      SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
2847
      SExprSupp*                pChildSup = &pChildOp->exprSupp;
2848 2849 2850 2851
      if (!hasIntervalWindow(&pChInfo->aggSup, pWinRes->ts, pWinRes->groupId)) {
        continue;
      }
      find = true;
2852
      SResultRow* pChResult = NULL;
2853 2854 2855
      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &ParentWin, true, &pChResult, pWinRes->groupId,
                             pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup,
                             pTaskInfo);
2856
      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo);
5
54liuyao 已提交
2857
    }
2858 2859
    if (find && pUpdated) {
      saveResultRow(pCurResult, pWinRes->groupId, pUpdated);
D
dapan1121 已提交
2860
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur);
2861
    }
5
54liuyao 已提交
2862 2863 2864 2865 2866
  }
}

bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
  SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId);
2867 2868
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
                                                            GET_RES_WINDOW_KEY_LEN(sizeof(int64_t)));
5
54liuyao 已提交
2869 2870 2871
  return p1 == NULL;
}

L
Liu Jicong 已提交
2872 2873 2874 2875
int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,
                        STimeWindow* pNextWin) {
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889
  int32_t prevEndPos = forwardRows - 1 + startPos;
  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
}

void addPullWindow(SHashObj* pMap, SWinRes* pWinRes, int32_t size) {
  SArray* childIds = taosArrayInit(8, sizeof(int32_t));
  for (int32_t i = 0; i < size; i++) {
    taosArrayPush(childIds, &i);
  }
  taosHashPut(pMap, pWinRes, sizeof(SWinRes), &childIds, sizeof(void*));
}

static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

5
54liuyao 已提交
2890 2891 2892 2893 2894 2895
STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
  return w;
}

5
54liuyao 已提交
2896
static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
5
54liuyao 已提交
2897
                           SHashObj* pUpdatedMap) {
5
54liuyao 已提交
2898
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
X
Xiaoyu Wang 已提交
2899 2900
  SResultRowInfo*                   pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*                    pTaskInfo = pOperatorInfo->pTaskInfo;
2901 2902
  SExprSupp*                        pSup = &pOperatorInfo->exprSupp;
  int32_t                           numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
2903 2904 2905 2906 2907
  int32_t                           step = 1;
  bool                              ascScan = true;
  TSKEY*                            tsCols = NULL;
  SResultRow*                       pResult = NULL;
  int32_t                           forwardRows = 0;
5
54liuyao 已提交
2908

5
54liuyao 已提交
2909 2910 2911
  ASSERT(pSDataBlock->pDataBlock != NULL);
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
2912

X
Xiaoyu Wang 已提交
2913 2914
  int32_t     startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
5
54liuyao 已提交
2915 2916 2917 2918 2919 2920
  STimeWindow nextWin = {0};
  if (IS_FINAL_OP(pInfo)) {
    nextWin = getFinalTimeWindow(ts, &pInfo->interval);
  } else {
    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order);
  }
5
54liuyao 已提交
2921
  while (1) {
5
54liuyao 已提交
2922
    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
5
54liuyao 已提交
2923
    if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
5
54liuyao 已提交
2924 2925 2926 2927 2928 2929 2930
      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
      if (startPos < 0) {
        break;
      }
      continue;
    }
    if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) {
L
Liu Jicong 已提交
2931 2932 2933 2934 2935
      bool    ignore = true;
      SWinRes winRes = {
          .ts = nextWin.skey,
          .groupId = tableGroupId,
      };
5
54liuyao 已提交
2936 2937 2938 2939
      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
      if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) {
        SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId};
        // add pull data request
5
54liuyao 已提交
2940
        savePullWindow(&pull, pInfo->pPullWins);
5
54liuyao 已提交
2941 2942 2943
        int32_t size = taosArrayGetSize(pInfo->pChildren);
        addPullWindow(pInfo->pPullDataMap, &winRes, size);
        qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size);
5
54liuyao 已提交
2944 2945 2946
      } else {
        int32_t index = -1;
        SArray* chArray = NULL;
5
54liuyao 已提交
2947
        int32_t chId = 0;
5
54liuyao 已提交
2948
        if (chIds) {
L
Liu Jicong 已提交
2949
          chArray = *(void**)chIds;
5
54liuyao 已提交
2950
          chId = getChildIndex(pSDataBlock);
5
54liuyao 已提交
2951 2952
          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
        }
L
Liu Jicong 已提交
2953
        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {
5
54liuyao 已提交
2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964
          ignore = false;
        }
      }

      if (ignore) {
        startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
        if (startPos < 0) {
          break;
        }
        continue;
      }
5
54liuyao 已提交
2965
    }
5
54liuyao 已提交
2966

2967 2968
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx,
                                          numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2969 2970 2971
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
5
54liuyao 已提交
2972

5
54liuyao 已提交
2973 2974 2975
    if (IS_FINAL_OP(pInfo)) {
      forwardRows = 1;
    } else {
H
Hongze Cheng 已提交
2976 2977
      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,
                                             NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
2978
    }
5
54liuyao 已提交
2979 2980
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
D
dapan1121 已提交
2981
      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
5
54liuyao 已提交
2982
    }
5
54liuyao 已提交
2983
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
2984 2985
    doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
                     pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
2986
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
2987
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
5
54liuyao 已提交
2988 2989 2990 2991 2992 2993 2994
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
    if (startPos < 0) {
      break;
    }
  }
}

5
54liuyao 已提交
2995 2996 2997 2998
static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) {
  taosHashClear(pInfo->aggSup.pResultRowHashTable);
  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
  cleanupResultRowInfo(&pInfo->binfo.resultRowInfo);
2999
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3000 3001
}

5
54liuyao 已提交
3002 3003 3004 3005
static void clearSpecialDataBlock(SSDataBlock* pBlock) {
  if (pBlock->info.rows <= 0) {
    return;
  }
5
54liuyao 已提交
3006 3007 3008
  blockDataCleanup(pBlock);
}

3009
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
5
54liuyao 已提交
3010 3011
  // ASSERT(pDest->info.capacity >= pSource->info.rows);
  blockDataEnsureCapacity(pDest, pSource->info.rows);
5
54liuyao 已提交
3012
  clearSpecialDataBlock(pDest);
5
54liuyao 已提交
3013 3014
  SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
  SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
3015

5
54liuyao 已提交
3016
  // copy timestamp column
3017 3018
  colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info);
  for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) {
5
54liuyao 已提交
3019 3020 3021
    SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
    colDataAppendNNULL(pCol, 0, pSource->info.rows);
  }
3022

5
54liuyao 已提交
3023
  pDest->info.rows = pSource->info.rows;
3024 3025
  pDest->info.groupId = pSource->info.groupId;
  pDest->info.type = pSource->info.type;
5
54liuyao 已提交
3026 3027 3028
  blockDataUpdateTsWindow(pDest, 0);
}

5
54liuyao 已提交
3029 3030 3031 3032 3033 3034
static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
  clearSpecialDataBlock(pBlock);
  int32_t size = taosArrayGetSize(array);
  if (size - (*pIndex) == 0) {
    return;
  }
L
Liu Jicong 已提交
3035
  blockDataEnsureCapacity(pBlock, size - (*pIndex));
5
54liuyao 已提交
3036
  ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
3037 3038 3039 3040 3041
  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
5
54liuyao 已提交
3042
  for (; (*pIndex) < size; (*pIndex)++) {
L
Liu Jicong 已提交
3043
    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));
5
54liuyao 已提交
3044 3045 3046
    colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
    colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
3047 3048
    colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
    colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
5
54liuyao 已提交
3049 3050 3051 3052 3053 3054 3055 3056 3057
    pBlock->info.rows++;
  }
  if ((*pIndex) == size) {
    *pIndex = 0;
    taosArrayClear(array);
  }
  blockDataUpdateTsWindow(pBlock, 0);
}

L
Liu Jicong 已提交
3058
void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
5
54liuyao 已提交
3059
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
L
Liu Jicong 已提交
3060
  TSKEY*           tsData = (TSKEY*)pStartCol->pData;
5
54liuyao 已提交
3061
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
L
Liu Jicong 已提交
3062 3063
  uint64_t*        groupIdData = (uint64_t*)pGroupCol->pData;
  int32_t          chId = getChildIndex(pBlock);
5
54liuyao 已提交
3064 3065
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SWinRes winRes = {.ts = tsData[i], .groupId = groupIdData[i]};
L
Liu Jicong 已提交
3066
    void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes));
5
54liuyao 已提交
3067
    if (chIds) {
L
Liu Jicong 已提交
3068
      SArray* chArray = *(SArray**)chIds;
5
54liuyao 已提交
3069 3070
      int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
      if (index != -1) {
5
54liuyao 已提交
3071
        qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId);
5
54liuyao 已提交
3072 3073 3074 3075 3076 3077 3078 3079 3080
        taosArrayRemove(chArray, index);
        if (taosArrayGetSize(chArray) == 0) {
          // pull data is over
          taosHashRemove(pMap, &winRes, sizeof(SWinRes));
        }
      }
    }
  }
}
5
54liuyao 已提交
3081

5
54liuyao 已提交
3082 3083
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
  SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3084
  SOperatorInfo*                    downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
3085
  SArray*                           pUpdated = taosArrayInit(4, POINTER_BYTES);
5
54liuyao 已提交
3086 3087
  _hash_fn_t                        hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP);
  SHashObj*                         pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
5
54liuyao 已提交
3088
  TSKEY                             maxTs = INT64_MIN;
5
54liuyao 已提交
3089

3090 3091
  SExprSupp* pSup = &pOperator->exprSupp;

5
54liuyao 已提交
3092
  qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
L
Liu Jicong 已提交
3093

5
54liuyao 已提交
3094 3095 3096
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
3097 3098 3099 3100
    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
    if (pInfo->pPullDataRes->info.rows != 0) {
      // process the rest of the data
      ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3101
      printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3102 3103 3104
      return pInfo->pPullDataRes;
    }

5
54liuyao 已提交
3105
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3106
    if (pInfo->binfo.pRes->info.rows == 0) {
5
54liuyao 已提交
3107
      pOperator->status = OP_EXEC_DONE;
5
54liuyao 已提交
3108 3109 3110
      if (!IS_FINAL_OP(pInfo)) {
        // semi interval operator clear disk buffer
        clearStreamIntervalOperator(pInfo);
3111 3112
      } else {
        freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3113
      }
5
54liuyao 已提交
3114 3115
      return NULL;
    }
5
54liuyao 已提交
3116
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3117 3118
    return pInfo->binfo.pRes;
  } else {
5
54liuyao 已提交
3119 3120 3121
    if (!IS_FINAL_OP(pInfo)) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
      if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3122
        printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3123 3124
        return pInfo->binfo.pRes;
      }
5
54liuyao 已提交
3125 3126 3127 3128
    }
    if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
      pInfo->returnUpdate = false;
      ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3129
      printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3130 3131
      // process the rest of the data
      return pInfo->pUpdateRes;
5
54liuyao 已提交
3132
    }
3133 3134 3135
    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows != 0) {
      // process the rest of the data
5
54liuyao 已提交
3136
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
3137 3138
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
3139 3140 3141 3142 3143
  }

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
3144
      clearSpecialDataBlock(pInfo->pUpdateRes);
5
54liuyao 已提交
3145
      removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
5
54liuyao 已提交
3146
      pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
3147
      qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3148 3149
      break;
    }
5
54liuyao 已提交
3150
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv");
5
54liuyao 已提交
3151
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
3152
    maxTs = TMAX(maxTs, pBlock->info.watermark);
3153

L
Liu Jicong 已提交
3154
    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA ||
L
Liu Jicong 已提交
3155
        pBlock->info.type == STREAM_INVALID) {
5
54liuyao 已提交
3156 3157
      pInfo->binfo.pRes->info.type = pBlock->info.type;
    } else if (pBlock->info.type == STREAM_CLEAR) {
3158
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes));
3159
      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
5
54liuyao 已提交
3160
      if (IS_FINAL_OP(pInfo)) {
L
Liu Jicong 已提交
3161 3162
        int32_t                           childIndex = getChildIndex(pBlock);
        SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
3163
        SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
L
Liu Jicong 已提交
3164
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;
3165

3166 3167 3168
        doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildSup->numOfExprs, pBlock, NULL);
        rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs,
                              pOperator->pTaskInfo, NULL);
5
54liuyao 已提交
3169 3170
        taosArrayDestroy(pUpWins);
        continue;
3171
      }
5
54liuyao 已提交
3172
      removeResults(pUpWins, pUpdatedMap);
3173 3174
      copyDataBlock(pInfo->pUpdateRes, pBlock);
      // copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
3175
      pInfo->returnUpdate = true;
3176
      taosArrayDestroy(pUpWins);
5
54liuyao 已提交
3177
      break;
3178 3179 3180 3181 3182 3183 3184 3185 3186
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval);
      if (IS_FINAL_OP(pInfo)) {
        int32_t                           childIndex = getChildIndex(pBlock);
        SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;
        doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval);
        rebuildIntervalWindow(pInfo, pSup, pInfo->pDelWins, pInfo->binfo.pRes->info.groupId,
3187
                              pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated);
3188 3189
        continue;
      }
5
54liuyao 已提交
3190
      removeResults(pInfo->pDelWins, pUpdatedMap);
3191
      break;
5
54liuyao 已提交
3192
    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
5
54liuyao 已提交
3193
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
5
54liuyao 已提交
3194
      continue;
5
54liuyao 已提交
3195
    } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
3196
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes));
5
54liuyao 已提交
3197
      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
5
54liuyao 已提交
3198
      removeResults(pUpWins, pUpdatedMap);
5
54liuyao 已提交
3199 3200 3201 3202 3203
      taosArrayDestroy(pUpWins);
      if (taosArrayGetSize(pUpdated) > 0) {
        break;
      }
      continue;
L
Liu Jicong 已提交
3204 3205
    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) {
      processPullOver(pBlock, pInfo->pPullDataMap);
5
54liuyao 已提交
3206
      continue;
5
54liuyao 已提交
3207
    }
5
54liuyao 已提交
3208

5
54liuyao 已提交
3209 3210 3211 3212
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
3213
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
5
54liuyao 已提交
3214
    doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
5
54liuyao 已提交
3215
    if (IS_FINAL_OP(pInfo)) {
S
shenglian zhou 已提交
3216
      int32_t chIndex = getChildIndex(pBlock);
5
54liuyao 已提交
3217 3218 3219 3220 3221 3222 3223
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
        SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
        if (!pChildOp) {
          longjmp(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
        }
3224 3225
        SStreamFinalIntervalOperatorInfo* pTmpInfo = pChildOp->info;
        pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3226
        taosArrayPush(pInfo->pChildren, &pChildOp);
5
54liuyao 已提交
3227
        qDebug("===stream===add child, id:%d", chIndex);
5
54liuyao 已提交
3228
      }
S
shenglian zhou 已提交
3229
      SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
3230
      SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
3231
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
5
54liuyao 已提交
3232 3233 3234
      doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL);
    }
  }
S
shenglian zhou 已提交
3235

5
54liuyao 已提交
3236
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
3237
  if (IS_FINAL_OP(pInfo)) {
3238
    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap,
5
54liuyao 已提交
3239
                        pUpdatedMap, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3240
    closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
3241 3242
  } else {
    pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;
5
54liuyao 已提交
3243 3244
  }

5
54liuyao 已提交
3245 3246 3247 3248 3249 3250 3251
  void* pIte = NULL;
  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
    taosArrayPush(pUpdated, pIte);
  }
  taosHashCleanup(pUpdatedMap);
  taosArraySort(pUpdated, resultrowComparAsc);

3252
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
3253 3254
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
3255 3256 3257 3258 3259

  doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
  if (pInfo->pPullDataRes->info.rows != 0) {
    // process the rest of the data
    ASSERT(IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3260
    printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3261 3262 3263
    return pInfo->pPullDataRes;
  }

5
54liuyao 已提交
3264
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
3265
  if (pInfo->binfo.pRes->info.rows != 0) {
5
54liuyao 已提交
3266
    printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3267 3268 3269 3270 3271 3272
    return pInfo->binfo.pRes;
  }

  if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
    pInfo->returnUpdate = false;
    ASSERT(!IS_FINAL_OP(pInfo));
5
54liuyao 已提交
3273
    printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
5
54liuyao 已提交
3274 3275 3276
    // process the rest of the data
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
3277

3278 3279 3280
  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
  if (pInfo->pDelRes->info.rows != 0) {
    // process the rest of the data
5
54liuyao 已提交
3281
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
3282 3283
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
3284 3285 3286
  return NULL;
}

3287
SSDataBlock* createSpecialDataBlock(EStreamType type) {
5
54liuyao 已提交
3288 3289 3290 3291
  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
  pBlock->info.hasVarCol = false;
  pBlock->info.groupId = 0;
  pBlock->info.rows = 0;
3292
  pBlock->info.type = type;
L
Liu Jicong 已提交
3293 3294
  pBlock->info.rowSize =
      sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY);
3295
  pBlock->info.watermark = INT64_MIN;
5
54liuyao 已提交
3296

5
54liuyao 已提交
3297
  pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData));
5
54liuyao 已提交
3298 3299 3300 3301 3302 3303 3304 3305 3306 3307
  SColumnInfoData infoData = {0};
  infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP;
  infoData.info.bytes = sizeof(TSKEY);
  // window start ts
  taosArrayPush(pBlock->pDataBlock, &infoData);
  // window end ts
  taosArrayPush(pBlock->pDataBlock, &infoData);

  infoData.info.type = TSDB_DATA_TYPE_UBIGINT;
  infoData.info.bytes = sizeof(uint64_t);
3308 3309 3310 3311 3312 3313 3314 3315
  // uid
  taosArrayPush(pBlock->pDataBlock, &infoData);
  // group id
  taosArrayPush(pBlock->pDataBlock, &infoData);

  // calculate start ts
  taosArrayPush(pBlock->pDataBlock, &infoData);
  // calculate end ts
5
54liuyao 已提交
3316 3317 3318
  taosArrayPush(pBlock->pDataBlock, &infoData);

  return pBlock;
5
54liuyao 已提交
3319 3320
}

S
shenglian zhou 已提交
3321 3322 3323
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                     SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
  SIntervalPhysiNode*               pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
5
54liuyao 已提交
3324
  SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo));
S
shenglian zhou 已提交
3325
  SOperatorInfo*                    pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3326 3327 3328
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
3329

3330
  pOperator->pTaskInfo = pTaskInfo;
5
54liuyao 已提交
3331
  pInfo->order = TSDB_ORDER_ASC;
S
shenglian zhou 已提交
3332 3333 3334 3335 3336 3337 3338 3339
  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                .sliding = pIntervalPhyNode->sliding,
                                .intervalUnit = pIntervalPhyNode->intervalUnit,
                                .slidingUnit = pIntervalPhyNode->slidingUnit,
                                .offset = pIntervalPhyNode->offset,
                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pIntervalPhyNode->window.watermark,
5
54liuyao 已提交
3340 3341
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
S
shenglian zhou 已提交
3342
  };
3343
  ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY);
5
54liuyao 已提交
3344 3345
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
3346
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3347 3348 3349 3350 3351 3352 3353 3354 3355
  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

S
shenglian zhou 已提交
3356 3357
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5
54liuyao 已提交
3358
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
3359 3360 3361 3362

  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);

3363
  ASSERT(numOfCols > 0);
3364
  increaseTs(pOperator->exprSupp.pCtx);
5
54liuyao 已提交
3365 3366 3367 3368
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
3369
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3370 3371
  pInfo->pChildren = NULL;
  if (numOfChild > 0) {
3372
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
5
54liuyao 已提交
3373 3374 3375
    for (int32_t i = 0; i < numOfChild; i++) {
      SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp) {
3376 3377
        SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
        pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
5
54liuyao 已提交
3378 3379 3380 3381 3382 3383
        taosArrayPush(pInfo->pChildren, &pChildOp);
        continue;
      }
      goto _error;
    }
  }
3384
  pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
5
54liuyao 已提交
3385
  blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
5
54liuyao 已提交
3386 3387
  pInfo->returnUpdate = false;

3388
  pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
5
54liuyao 已提交
3389

5
54liuyao 已提交
3390 3391 3392 3393
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
    pInfo->isFinal = true;
    pOperator->name = "StreamFinalIntervalOperator";
  } else {
5
54liuyao 已提交
3394
    // semi interval operator does not catch result
5
54liuyao 已提交
3395 3396 3397 3398
    pInfo->isFinal = false;
    pOperator->name = "StreamSemiIntervalOperator";
  }

5
54liuyao 已提交
3399
  if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
5
54liuyao 已提交
3400 3401
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }
5
54liuyao 已提交
3402 3403 3404 3405
  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
  pInfo->pullIndex = 0;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
3406
  pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
5
54liuyao 已提交
3407
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
3408
  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3409 3410
  pInfo->delIndex = 0;
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes));
5
54liuyao 已提交
3411
  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
5
54liuyao 已提交
3412

5
54liuyao 已提交
3413
  pOperator->operatorType = pPhyNode->type;
5
54liuyao 已提交
3414 3415
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
3416 3417
  pOperator->exprSupp.pExprInfo = pExprInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
5
54liuyao 已提交
3418 3419
  pOperator->info = pInfo;

S
shenglian zhou 已提交
3420 3421 3422
  pOperator->fpSet =
      createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
3423
  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
3424
    initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup);
3425
  }
5
54liuyao 已提交
3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
  destroyStreamFinalIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
5
54liuyao 已提交
3439
}
5
54liuyao 已提交
3440 3441 3442

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
3443
  void** pIte = NULL;
3444
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
3445
    SArray* pWins = (SArray*)(*pIte);
3446 3447 3448
    taosArrayDestroy(pWins);
  }
  taosHashCleanup(pSup->pResultRows);
5
54liuyao 已提交
3449
  destroyDiskbasedBuf(pSup->pResultBuf);
3450
  blockDataDestroy(pSup->pScanBlock);
5
54liuyao 已提交
3451 3452
}

5
54liuyao 已提交
3453 3454 3455 3456
void destroyStateWinInfo(void* ptr) {
  if (ptr == NULL) {
    return;
  }
3457
  SStateWindowInfo* pWin = (SStateWindowInfo*)ptr;
5
54liuyao 已提交
3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472
  taosMemoryFreeClear(pWin->stateKey.pData);
}

void destroyStateStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
  void** pIte = NULL;
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
    SArray* pWins = (SArray*)(*pIte);
    taosArrayDestroyEx(pWins, (FDelete)destroyStateWinInfo);
  }
  taosHashCleanup(pSup->pResultRows);
  destroyDiskbasedBuf(pSup->pResultBuf);
  blockDataDestroy(pSup->pScanBlock);
}

5
54liuyao 已提交
3473 3474
void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;
3475
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
3476 3477
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
3478 3479 3480
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
3481
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
3482 3483 3484 3485 3486
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
      destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput);
      taosMemoryFreeClear(pChild);
    }
  }
5
54liuyao 已提交
3487 3488 3489 3490
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  blockDataDestroy(pInfo->pWinBlock);
  blockDataDestroy(pInfo->pUpdateRes);
5
54liuyao 已提交
3491 3492
  destroySqlFunctionCtx(pInfo->pDummyCtx, 0);
  taosHashCleanup(pInfo->pStDeleted);
3493

D
dapan1121 已提交
3494
  taosMemoryFreeClear(param);
5
54liuyao 已提交
3495 3496
}

3497 3498
int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,
                        SSDataBlock* pResultBlock) {
3499 3500 3501 3502 3503
  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
  if (code != TSDB_CODE_SUCCESS) {
    return code;
  }

3504
  initBasicInfo(pBasicInfo, pResultBlock);
3505

5
54liuyao 已提交
3506
  for (int32_t i = 0; i < numOfCols; ++i) {
3507
    pSup->pCtx[i].pBuf = NULL;
5
54liuyao 已提交
3508
  }
3509

3510
  ASSERT(numOfCols > 0);
3511
  increaseTs(pSup->pCtx);
5
54liuyao 已提交
3512 3513 3514 3515 3516 3517 3518 3519
  return TSDB_CODE_SUCCESS;
}

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {
  for (int i = 0; i < nums; i++) {
    pDummy[i].functionId = pCtx[i].functionId;
  }
}
5
54liuyao 已提交
3520

X
Xiaoyu Wang 已提交
3521 3522
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
                    uint8_t type) {
5
54liuyao 已提交
3523
  ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
3524
  SStreamScanInfo* pScanInfo = downstream->info;
X
Xiaoyu Wang 已提交
3525
  pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
5
54liuyao 已提交
3526
  pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
5
54liuyao 已提交
3527 3528
}

3529 3530
int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx,
                                int32_t numOfOutput) {
3531 3532 3533
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo));
}

3534 3535 3536 3537 3538 3539 3540
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                  SExecTaskInfo* pTaskInfo) {
  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
  int32_t                        numOfCols = 0;
  SExprInfo*                     pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock*                   pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;
X
Xiaoyu Wang 已提交
3541
  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
3542
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3543 3544 3545 3546
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

3547 3548
  pOperator->pTaskInfo = pTaskInfo;

3549
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
3550 3551 3552 3553 3554 3555 3556 3557
  if (pSessionNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }
3558
  SExprSupp* pSup = &pOperator->exprSupp;
5
54liuyao 已提交
3559

3560
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
3561 3562 3563
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
3564

3565
  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
3566 3567 3568
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
X
Xiaoyu Wang 已提交
3569

5
54liuyao 已提交
3570 3571 3572 3573
  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }
3574
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
3575

3576 3577
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType, .maxTs = INT64_MIN};
H
Haojun Liao 已提交
3578 3579

  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
3580 3581
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

5
54liuyao 已提交
3582 3583 3584 3585
  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  if (pSessionNode->window.pTsEnd) {
    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;
  }
3586
  pInfo->gap = pSessionNode->gap;
5
54liuyao 已提交
3587 3588 3589 3590 3591
  pInfo->binfo.pRes = pResBlock;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
3592
  // pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
3593 3594
  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete
3595
  pInfo->pChildren = NULL;
5
54liuyao 已提交
3596 3597
  pInfo->isFinal = false;
  pInfo->pPhyNode = pPhyNode;
5
54liuyao 已提交
3598
  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
5
54liuyao 已提交
3599
  pInfo->returnDelete = false;
5
54liuyao 已提交
3600 3601

  pOperator->name = "StreamSessionWindowAggOperator";
3602
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
5
54liuyao 已提交
3603 3604
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
3605 3606
  pOperator->exprSupp.pExprInfo = pExprInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
5
54liuyao 已提交
3607
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
3608 3609 3610
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
3611 3612 3613 3614
  if (downstream) {
    initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType);
    code = appendDownstream(pOperator, &downstream, 1);
  }
5
54liuyao 已提交
3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628
  return pOperator;

_error:
  if (pInfo != NULL) {
    destroyStreamSessionAggOperatorInfo(pInfo, numOfCols);
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

int64_t getSessionWindowEndkey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
3629
  SArray*            pWinInfos = (SArray*)data;
5
54liuyao 已提交
3630 3631 3632
  SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index);
  return pWin->win.ekey;
}
5
54liuyao 已提交
3633 3634

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {
5
54liuyao 已提交
3635
  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {
5
54liuyao 已提交
3636 3637 3638 3639 3640
    return true;
  }
  return false;
}

3641
bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); }
5
54liuyao 已提交
3642

X
Xiaoyu Wang 已提交
3643 3644
static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) {
  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
5
54liuyao 已提交
3645 3646 3647 3648
  return taosArrayInsert(pWinInfos, index, &win);
}

static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) {
X
Xiaoyu Wang 已提交
3649
  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
5
54liuyao 已提交
3650 3651 3652
  return taosArrayPush(pWinInfos, &win);
}

3653
SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) {
3654
  void**  ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t));
3655 3656 3657
  SArray* pWinInfos = NULL;
  if (ite == NULL) {
    pWinInfos = taosArrayInit(1024, pAggSup->valueSize);
3658
    taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void*));
3659 3660 3661 3662 3663 3664
  } else {
    pWinInfos = *ite;
  }
  return pWinInfos;
}

5
54liuyao 已提交
3665 3666
// don't add new window
SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
3667
                                       int64_t gap, int32_t* pIndex) {
5
54liuyao 已提交
3668 3669 3670 3671 3672 3673 3674 3675
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    return NULL;
  }
  // find the first position which is smaller than the key
3676
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isInWindow(pWin, startTs, gap)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isInWindow(pWin, startTs, gap)) {
      *pIndex = index + 1;
      return pWin;
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
3692
      *pIndex = index + 1;
5
54liuyao 已提交
3693 3694 3695 3696 3697 3698 3699
      return pWin;
    }
  }

  return NULL;
}

3700 3701
SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
                                        int64_t gap, int32_t* pIndex) {
3702 3703 3704
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

5
54liuyao 已提交
3705 3706
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
5
54liuyao 已提交
3707
    *pIndex = 0;
5
54liuyao 已提交
3708
    return addNewSessionWindow(pWinInfos, startTs);
5
54liuyao 已提交
3709 3710
  }
  // find the first position which is smaller than the key
5
54liuyao 已提交
3711
  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
3712 3713 3714
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
5
54liuyao 已提交
3715
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3716 3717 3718 3719 3720 3721 3722
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
5
54liuyao 已提交
3723
    if (isInWindow(pWin, startTs, gap)) {
5
54liuyao 已提交
3724 3725
      *pIndex = index + 1;
      return pWin;
5
54liuyao 已提交
3726 3727 3728
    } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
      *pIndex = index;
      return pWin;
5
54liuyao 已提交
3729 3730 3731 3732 3733
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
5
54liuyao 已提交
3734
    return addNewSessionWindow(pWinInfos, startTs);
5
54liuyao 已提交
3735
  }
5
54liuyao 已提交
3736
  *pIndex = index + 1;
5
54liuyao 已提交
3737
  return insertNewSessionWindow(pWinInfos, startTs, index + 1);
5
54liuyao 已提交
3738 3739
}

5
54liuyao 已提交
3740
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId,int32_t rows,
3741
                                int32_t start, int64_t gap, SHashObj* pStDeleted) {
5
54liuyao 已提交
3742
  for (int32_t i = start; i < rows; ++i) {
3743
    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {
5
54liuyao 已提交
3744 3745
      return i - start;
    }
5
54liuyao 已提交
3746
    if (pWinInfo->win.skey > pStartTs[i]) {
5
54liuyao 已提交
3747
      if (pStDeleted && pWinInfo->isOutput) {
5
54liuyao 已提交
3748 3749
        SWinRes res = {.ts = pWinInfo->win.skey, .groupId = groupId};
        taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinRes));
5
54liuyao 已提交
3750 3751
        pWinInfo->isOutput = false;
      }
5
54liuyao 已提交
3752 3753 3754 3755 3756
      pWinInfo->win.skey = pStartTs[i];
    }
    pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pStartTs[i]);
    if (pEndTs) {
      pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pEndTs[i]);
5
54liuyao 已提交
3757 3758 3759 3760 3761
    }
  }
  return rows - start;
}

X
Xiaoyu Wang 已提交
3762
static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,
3763
                                  uint64_t groupId, int32_t numOfOutput, int32_t* rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3764
                                  SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) {
5
54liuyao 已提交
3765 3766
  assert(pWinInfo->win.skey <= pWinInfo->win.ekey);
  // too many time window in query
3767
  int32_t size = taosArrayGetSize(pAggSup->pCurWins);
3768
  if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH && size > MAX_INTERVAL_TIME_WINDOW) {
5
54liuyao 已提交
3769 3770
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
  }
X
Xiaoyu Wang 已提交
3771

5
54liuyao 已提交
3772
  if (pWinInfo->pos.pageId == -1) {
3773
    *pResult = getNewResultRow(pAggSup->pResultBuf, groupId, pAggSup->resultRowSize);
5
54liuyao 已提交
3774 3775 3776 3777 3778 3779 3780 3781 3782
    if (*pResult == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }
    initResultRow(*pResult);

    // add a new result set for a new group
    pWinInfo->pos.pageId = (*pResult)->pageId;
    pWinInfo->pos.offset = (*pResult)->offset;
  } else {
3783
    *pResult = getResultRowByPos(pAggSup->pResultBuf, &pWinInfo->pos, true);
5
54liuyao 已提交
3784 3785 3786 3787 3788 3789 3790 3791
    if (!(*pResult)) {
      qError("getResultRowByPos return NULL, TID:%s", GET_TASKID(pTaskInfo));
      return TSDB_CODE_FAILED;
    }
  }

  // set time window for current result
  (*pResult)->win = pWinInfo->win;
3792
  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
5
54liuyao 已提交
3793 3794 3795
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3796 3797 3798
static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup,
                                  SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock,
                                  SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3799
                                  int32_t numOutput, SOperatorInfo* pOperator) {
3800
  SExprSupp*     pSup = &pOperator->exprSupp;
3801 3802
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

X
Xiaoyu Wang 已提交
3803 3804
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId);
  TSKEY*           tsCols = (int64_t*)pColDataInfo->pData;
3805 3806
  int32_t          code = setWindowOutputBuf(pCurWin, pResult, pSup->pCtx, pSDataBlock->info.groupId, numOutput,
                                             pSup->rowEntryInfoOffset, pAggSup, pTaskInfo);
5
54liuyao 已提交
3807 3808 3809
  if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }
5
54liuyao 已提交
3810
  updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false);
3811
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols,
X
Xiaoyu Wang 已提交
3812
                   pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC);
5
54liuyao 已提交
3813 3814 3815
  SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pCurWin->pos.pageId);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pAggSup->pResultBuf, bufPage);
5
54liuyao 已提交
3816 3817 3818
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
3819 3820
static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                              SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
3821
                              int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3822
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3823
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3824 3825
}

X
Xiaoyu Wang 已提交
3826 3827
static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                                   SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex,
3828
                                   int32_t winRows, int32_t numOutput, SOperatorInfo* pOperator) {
X
Xiaoyu Wang 已提交
3829
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
3830
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
3831 3832
}

5
54liuyao 已提交
3833 3834
int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) {
  SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex);
X
Xiaoyu Wang 已提交
3835
  int32_t            size = taosArrayGetSize(pWinInfos);
5
54liuyao 已提交
3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i);
    if (!isInWindow(pCurWin, pWinInfo->win.skey, gap)) {
      return i - startIndex - 1;
    }
  }

  return size - startIndex - 1;
}

5
54liuyao 已提交
3847
void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId,
3848
                       int32_t numOfOutput, SHashObj* pStUpdated, SHashObj* pStDeleted, SOperatorInfo* pOperator) {
3849
  SExprSupp*     pSup = &pOperator->exprSupp;
3850 3851
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

3852
  SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex);
X
Xiaoyu Wang 已提交
3853
  SResultRow*        pCurResult = NULL;
3854
  setWindowOutputBuf(pCurWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3855
                     &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3856
  num += startIndex + 1;
3857
  ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins));
5
54liuyao 已提交
3858 3859
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < num; i++) {
3860
    SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i);
X
Xiaoyu Wang 已提交
3861
    SResultRow*        pWinResult = NULL;
3862
    setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
3863
                       &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
3864
    pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey);
3865
    compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo);
5
54liuyao 已提交
3866 3867
    taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition));
    if (pWinInfo->isOutput) {
5
54liuyao 已提交
3868 3869
      SWinRes res = {.ts = pWinInfo->win.skey, .groupId = groupId};
      taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinRes));
5
54liuyao 已提交
3870 3871
      pWinInfo->isOutput = false;
    }
3872
    taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
5
54liuyao 已提交
3873 3874
    SFilePage* tmpPage = getBufPage(pInfo->streamAggSup.pResultBuf, pWinInfo->pos.pageId);
    releaseBufPage(pInfo->streamAggSup.pResultBuf, tmpPage);
5
54liuyao 已提交
3875
  }
5
54liuyao 已提交
3876 3877 3878 3879
  SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pCurWin->pos.pageId);
  ASSERT(num > 0);
  setBufPageDirty(bufPage, true);
  releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
5
54liuyao 已提交
3880 3881
}

3882 3883
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
                                   SHashObj* pStDeleted, bool hasEndTs) {
X
Xiaoyu Wang 已提交
3884
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3885
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3886
  bool                           masterScan = true;
3887
  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
5
54liuyao 已提交
3888
  uint64_t                       groupId = pSDataBlock->info.groupId;
X
Xiaoyu Wang 已提交
3889 3890 3891 3892 3893
  int64_t                        gap = pInfo->gap;
  int64_t                        code = TSDB_CODE_SUCCESS;

  int32_t     step = 1;
  bool        ascScan = true;
5
54liuyao 已提交
3894 3895
  TSKEY*      startTsCols = NULL;
  TSKEY*      endTsCols = NULL;
5
54liuyao 已提交
3896
  SResultRow* pResult = NULL;
X
Xiaoyu Wang 已提交
3897
  int32_t     winRows = 0;
5
54liuyao 已提交
3898

5
54liuyao 已提交
3899 3900 3901 3902 3903 3904
  ASSERT(pSDataBlock->pDataBlock);
  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
  startTsCols = (int64_t*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = NULL;
  if (hasEndTs) {
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);
5
54liuyao 已提交
3905
  } else {
5
54liuyao 已提交
3906
    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
3907
  }
5
54liuyao 已提交
3908
  endTsCols = (int64_t*)pEndTsCol->pData;
X
Xiaoyu Wang 已提交
3909

5
54liuyao 已提交
3910
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
3911
  for (int32_t i = 0; i < pSDataBlock->info.rows;) {
5
54liuyao 已提交
3912
    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
3913 3914 3915
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
3916
    int32_t            winIndex = 0;
3917 3918
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex);
    winRows =
5
54liuyao 已提交
3919
        updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, groupId, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted);
3920
    code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
3921 3922 3923
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
5
54liuyao 已提交
3924

3925
    int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap);
5
54liuyao 已提交
3926
    if (winNum > 0) {
3927
      compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, pStDeleted, pOperator);
5
54liuyao 已提交
3928
    }
5
54liuyao 已提交
3929
    pCurWin->isClosed = false;
5
54liuyao 已提交
3930
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
5
54liuyao 已提交
3931 3932
      SWinRes value = {.ts = pCurWin->win.skey, .groupId = groupId};
      code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinRes));
5
54liuyao 已提交
3933 3934 3935 3936
      if (code != TSDB_CODE_SUCCESS) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
      pCurWin->isOutput = true;
5
54liuyao 已提交
3937 3938 3939 3940 3941
    }
    i += winRows;
  }
}

5
54liuyao 已提交
3942
void deleteWindow(SArray* pWinInfos, int32_t index, FDelete fp) {
5
54liuyao 已提交
3943
  ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos));
5
54liuyao 已提交
3944 3945 3946 3947
  if (fp) {
    void* ptr = taosArrayGet(pWinInfos, index);
    fp(ptr);
  }
5
54liuyao 已提交
3948 3949 3950
  taosArrayRemove(pWinInfos, index);
}

3951 3952
static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result,
                                FDelete fp) {
5
54liuyao 已提交
3953
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
3954
  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;
5
54liuyao 已提交
3955
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
3956
  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;
3957
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
3958
  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;
5
54liuyao 已提交
3959 3960
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    int32_t winIndex = 0;
3961 3962
    while (1) {
      SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex);
5
54liuyao 已提交
3963 3964 3965
      if (!pCurWin) {
        break;
      }
5
54liuyao 已提交
3966
      deleteWindow(pAggSup->pCurWins, winIndex, fp);
5
54liuyao 已提交
3967
      if (result) {
5
54liuyao 已提交
3968
        pCurWin->groupId = gpDatas[i];
5
54liuyao 已提交
3969 3970 3971 3972 3973 3974
        taosArrayPush(result, pCurWin);
      }
    }
  }
}

3975 3976
static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock, int32_t tsIndex,
                                  int32_t numOfOutput, int64_t gap, SArray* result) {
5
54liuyao 已提交
3977
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
3978 3979
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
  int32_t          step = 0;
5
54liuyao 已提交
3980
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
3981
    int32_t            winIndex = 0;
3982
    SResultWindowInfo* pCurWin =
5
54liuyao 已提交
3983 3984
        getCurSessionWindow(pAggSup, tsCols[i], INT64_MIN, pBlock->info.groupId, gap, &winIndex);
    if (!pCurWin || pCurWin->pos.pageId == -1) {
5
54liuyao 已提交
3985
      // window has been closed.
5
54liuyao 已提交
3986
      step = 1;
5
54liuyao 已提交
3987 3988
      continue;
    }
5
54liuyao 已提交
3989
    step = updateSessionWindowInfo(pCurWin, tsCols, NULL, 0, pBlock->info.rows, i, gap, NULL);
5
54liuyao 已提交
3990
    ASSERT(isInWindow(pCurWin, tsCols[i], gap));
3991
    doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
3992 3993 3994
    if (result) {
      taosArrayPush(result, pCurWin);
    }
5
54liuyao 已提交
3995 3996 3997
  }
}

5
54liuyao 已提交
3998
static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) {
X
Xiaoyu Wang 已提交
3999
  void*  pData = NULL;
5
54liuyao 已提交
4000
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
4001
  while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) {
5
54liuyao 已提交
4002 4003 4004 4005 4006 4007
    void* key = taosHashGetKey(pData, &keyLen);
    ASSERT(keyLen == sizeof(SResultRowPosition));
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    if (pos == NULL) {
      return TSDB_CODE_QRY_OUT_OF_MEMORY;
    }
5
54liuyao 已提交
4008
    pos->groupId = ((SWinRes*)pData)->groupId;
5
54liuyao 已提交
4009
    pos->pos = *(SResultRowPosition*)key;
5
54liuyao 已提交
4010
    *(int64_t*)pos->key = ((SWinRes*)pData)->ts;
5
54liuyao 已提交
4011 4012
    taosArrayPush(pUpdated, &pos);
  }
5
54liuyao 已提交
4013
  taosArraySort(pUpdated, resultrowComparAsc);
5
54liuyao 已提交
4014 4015 4016 4017 4018
  return TSDB_CODE_SUCCESS;
}

void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
  blockDataCleanup(pBlock);
4019 4020 4021 4022 4023
  int32_t size = taosHashGetSize(pStDeleted);
  if (size == 0) {
    return;
  }
  blockDataEnsureCapacity(pBlock, size);
5
54liuyao 已提交
4024
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
4025
  while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
5
54liuyao 已提交
4026 4027 4028 4029 4030
    SWinRes* res = *Ite;
    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
    colDataAppend(pTsCol, pBlock->info.rows, (const char*)&res->ts, false);
    SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
    colDataAppend(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false);
5
54liuyao 已提交
4031 4032 4033 4034 4035 4036 4037 4038 4039 4040
    pBlock->info.rows += 1;
    if (pBlock->info.rows + 1 >= pBlock->info.capacity) {
      break;
    }
  }
  if ((*Ite) == NULL) {
    taosHashClear(pStDeleted);
  }
}

X
Xiaoyu Wang 已提交
4041
static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId,
4042
                              int32_t numOfOutput, SOperatorInfo* pOperator) {
4043
  SExprSupp*     pSup = &pOperator->exprSupp;
4044 4045
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

4046 4047
  int32_t size = taosArrayGetSize(pWinArray);
  ASSERT(pInfo->pChildren);
4048

4049 4050
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i);
X
Xiaoyu Wang 已提交
4051
    SResultRow*        pCurResult = NULL;
4052
    setWindowOutputBuf(pParentWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
4053
                       &pInfo->streamAggSup, pTaskInfo);
4054 4055
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
    for (int32_t j = 0; j < numOfChildren; j++) {
X
Xiaoyu Wang 已提交
4056
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
4057
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
4058
      SArray*                        pChWins = getWinInfos(&pChInfo->streamAggSup, groupId);
X
Xiaoyu Wang 已提交
4059 4060
      int32_t                        chWinSize = taosArrayGetSize(pChWins);
      int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
4061
      if (index < 0) {
4062
        index = 0;
5
54liuyao 已提交
4063 4064
      }
      for (int32_t k = index; k < chWinSize; k++) {
5
54liuyao 已提交
4065 4066
        SResultWindowInfo* pChWin = taosArrayGet(pChWins, k);
        if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) {
4067
          SResultRow* pChResult = NULL;
5
54liuyao 已提交
4068
          setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
4069 4070
                             pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
          compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo);
5
54liuyao 已提交
4071 4072
          SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
          releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage);
4073
          continue;
5
54liuyao 已提交
4074 4075
        } else if (!pChWin->isClosed) {
          break;
4076 4077 4078
        }
      }
    }
5
54liuyao 已提交
4079 4080 4081 4082
    SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId);
    ASSERT(size > 0);
    setBufPageDirty(bufPage, true);
    releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
4083 4084 4085
  }
}

X
Xiaoyu Wang 已提交
4086
typedef SResultWindowInfo* (*__get_win_info_)(void*);
5
54liuyao 已提交
4087 4088
SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
5
54liuyao 已提交
4089

4090
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn,
5
54liuyao 已提交
4091
                           bool delete, FDelete fp) {
5
54liuyao 已提交
4092
  // Todo(liuyao) save window to tdb
4093
  void** pIte = NULL;
5
54liuyao 已提交
4094
  size_t keyLen = 0;
4095
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
5
54liuyao 已提交
4096
    uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen);
4097 4098
    SArray*   pWins = (SArray*)(*pIte);
    int32_t   size = taosArrayGetSize(pWins);
4099 4100 4101
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
5
54liuyao 已提交
4102
      if (isCloseWindow(&pSeWin->win, pTwSup)) {
4103 4104
        if (!pSeWin->isClosed) {
          pSeWin->isClosed = true;
5
54liuyao 已提交
4105
          if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {
5
54liuyao 已提交
4106
            int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed);
5
54liuyao 已提交
4107 4108 4109
            if (code != TSDB_CODE_SUCCESS) {
              return code;
            }
4110 4111
            pSeWin->isOutput = true;
          }
5
54liuyao 已提交
4112
          if (delete) {
5
54liuyao 已提交
4113
            deleteWindow(pWins, i, fp);
5
54liuyao 已提交
4114 4115 4116
            i--;
            size = taosArrayGetSize(pWins);
          }
5
54liuyao 已提交
4117
        }
4118
        continue;
5
54liuyao 已提交
4119
      }
4120
      break;
5
54liuyao 已提交
4121 4122 4123 4124 4125
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
4126
static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs, bool delete, FDelete fp) {
5
54liuyao 已提交
4127 4128 4129 4130 4131
  int32_t size = taosArrayGetSize(pChildren);
  for (int32_t i = 0; i < size; i++) {
    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);
    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4132
    closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL, getResWinForSession, delete, fp);
5
54liuyao 已提交
4133 4134 4135
  }
}

4136
int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) {
4137
  void** pIte = NULL;
4138
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
4139
    SArray* pWins = (SArray*)(*pIte);
4140 4141 4142 4143 4144 4145 4146 4147
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (!pSeWin->isClosed) {
        int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed);
        pSeWin->isOutput = true;
      }
5
54liuyao 已提交
4148 4149 4150 4151 4152
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
4153 4154 4155 4156
static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) {
  int32_t size = taosArrayGetSize(pResWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i);
5
54liuyao 已提交
4157 4158
    SWinRes res = {.ts = pWinInfo->win.skey, .groupId = pWinInfo->groupId};
    taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinRes));
5
54liuyao 已提交
4159 4160 4161
  }
}

5
54liuyao 已提交
4162
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
5
54liuyao 已提交
4163
  SExprSupp*                     pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4164
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4165
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
5
54liuyao 已提交
4166 4167 4168 4169
  TSKEY                          maxTs = INT64_MIN;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
4170 4171
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4172
      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4173 4174
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
4175
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
4176
    if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
5
54liuyao 已提交
4177 4178
      doSetOperatorCompleted(pOperator);
    }
5
54liuyao 已提交
4179
    printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4180 4181 4182
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
4183 4184
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
4185
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
4186
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
4187 4188 4189 4190 4191
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4192
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv");
4193

5
54liuyao 已提交
4194
    if (pBlock->info.type == STREAM_CLEAR) {
X
Xiaoyu Wang 已提交
4195
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
4196 4197
      doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, 0,
                            pWins);
5
54liuyao 已提交
4198 4199
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
X
Xiaoyu Wang 已提交
4200
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
4201
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
4202
        doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, 0, pChildOp->exprSupp.numOfExprs,
5
54liuyao 已提交
4203
                              0, NULL);
4204
        rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
4205 4206
      }
      taosArrayDestroy(pWins);
5
54liuyao 已提交
4207
      continue;
5
54liuyao 已提交
4208 4209 4210
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      // gap must be 0
5
54liuyao 已提交
4211
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL);
5
54liuyao 已提交
4212 4213 4214 4215 4216
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
        // gap must be 0
5
54liuyao 已提交
4217
        doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL);
5
54liuyao 已提交
4218 4219 4220 4221 4222
        rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
      }
      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
      taosArrayDestroy(pWins);
      continue;
4223
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4224
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
5
54liuyao 已提交
4225
      continue;
5
54liuyao 已提交
4226
    }
5
54liuyao 已提交
4227

5
54liuyao 已提交
4228 4229 4230 4231
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4232
    // the pDataBlock are always the same one, no need to call this again
4233
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4234 4235 4236 4237 4238 4239
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo));
    if (IS_FINAL_OP(pInfo)) {
      int32_t chIndex = getChildIndex(pBlock);
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
4240 4241
        SOperatorInfo* pChildOp =
            createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
5
54liuyao 已提交
4242 4243 4244 4245 4246
        if (!pChildOp) {
          longjmp(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
        }
        taosArrayPush(pInfo->pChildren, &pChildOp);
      }
4247
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
4248 4249
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true);
4250
    }
5
54liuyao 已提交
4251
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
4252
    maxTs = TMAX(maxTs, pBlock->info.watermark);
5
54liuyao 已提交
4253
  }
5
54liuyao 已提交
4254 4255

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
4256 4257
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
4258

4259
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession,
5
54liuyao 已提交
4260 4261
                     pInfo->ignoreExpiredData, NULL);
  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, NULL);
5
54liuyao 已提交
4262 4263 4264
  copyUpdateResult(pStUpdated, pUpdated);
  taosHashCleanup(pStUpdated);

4265
  finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4266 4267 4268 4269
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4270
    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4271 4272 4273
    return pInfo->pDelRes;
  }
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
4274
  printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
5
54liuyao 已提交
4275 4276 4277 4278
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
4279
  void** pIte = NULL;
5
54liuyao 已提交
4280
  while ((pIte = taosHashIterate(pInfo->streamAggSup.pResultRows, pIte)) != NULL) {
4281
    SArray* pWins = (SArray*)(*pIte);
5
54liuyao 已提交
4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      SResultWindowInfo* pWin = (SResultWindowInfo*)taosArrayGet(pWins, i);
      pWin->pos.pageId = -1;
      pWin->pos.offset = -1;
    }
  }
  clearDiskbasedBuf(pInfo->streamAggSup.pResultBuf);
  cleanupResultRowInfo(&pInfo->binfo.resultRowInfo);
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
}

static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) {
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pWin = taosArrayGet(pWins, i);
    taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition));
  }
}

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
  TSKEY                          maxTs = INT64_MIN;
  SExprSupp*                     pSup = &pOperator->exprSupp;
4307

5
54liuyao 已提交
4308 4309 4310
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
5
54liuyao 已提交
4311 4312
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
    if (pBInfo->pRes->info.rows > 0) {
5
54liuyao 已提交
4313
      printDataBlock(pBInfo->pRes, "sems session");
5
54liuyao 已提交
4314 4315 4316 4317 4318 4319
      return pBInfo->pRes;
    }

    // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) {
      pInfo->returnDelete = true;
5
54liuyao 已提交
4320
      printDataBlock(pInfo->pDelRes, "sems session");
5
54liuyao 已提交
4321 4322
      return pInfo->pDelRes;
    }
5
54liuyao 已提交
4323 4324

    if (pInfo->pUpdateRes->info.rows > 0) {
5
54liuyao 已提交
4325 4326
      // process the rest of the data
      pOperator->status = OP_OPENED;
5
54liuyao 已提交
4327
      printDataBlock(pInfo->pUpdateRes, "sems session");
5
54liuyao 已提交
4328 4329
      return pInfo->pUpdateRes;
    }
5
54liuyao 已提交
4330 4331 4332 4333
    // semi interval operator clear disk buffer
    clearStreamSessionOperator(pInfo);
    pOperator->status = OP_EXEC_DONE;
    return NULL;
5
54liuyao 已提交
4334 4335 4336 4337 4338 4339 4340 4341 4342
  }

  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  SOperatorInfo* downstream = pOperator->pDownstream[0];
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
4343
      clearSpecialDataBlock(pInfo->pUpdateRes);
5
54liuyao 已提交
4344 4345 4346
      break;
    }

5
54liuyao 已提交
4347
    if (pBlock->info.type == STREAM_CLEAR) {
5
54liuyao 已提交
4348
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
5
54liuyao 已提交
4349
      doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, 0, pSup->numOfExprs, 0, pWins);
5
54liuyao 已提交
4350 4351 4352 4353
      removeSessionResults(pStUpdated, pWins);
      taosArrayDestroy(pWins);
      copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
      break;
5
54liuyao 已提交
4354 4355
    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
      // gap must be 0
5
54liuyao 已提交
4356
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL, NULL);
5
54liuyao 已提交
4357 4358 4359
      copyDataBlock(pInfo->pDelRes, pBlock);
      pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
      break;
5
54liuyao 已提交
4360 4361 4362 4363 4364
    } else if (pBlock->info.type == STREAM_GET_ALL) {
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
      continue;
    }

5
54liuyao 已提交
4365 4366 4367 4368
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
5
54liuyao 已提交
4369 4370 4371 4372 4373 4374 4375
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, false);
    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
4376
  pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;
5
54liuyao 已提交
4377 4378 4379 4380 4381
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
  // semi operator
  // closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
  //                    getResWinForSession);
5
54liuyao 已提交
4382
  copyUpdateResult(pStUpdated, pUpdated);
5
54liuyao 已提交
4383
  taosHashCleanup(pStUpdated);
5
54liuyao 已提交
4384

4385 4386
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4387
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
5
54liuyao 已提交
4388
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
5
54liuyao 已提交
4389 4390 4391

  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
  if (pBInfo->pRes->info.rows > 0) {
5
54liuyao 已提交
4392
    printDataBlock(pBInfo->pRes, "sems session");
5
54liuyao 已提交
4393 4394 4395 4396 4397 4398
    return pBInfo->pRes;
  }

  // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) {
    pInfo->returnDelete = true;
5
54liuyao 已提交
4399
    printDataBlock(pInfo->pDelRes, "sems session");
5
54liuyao 已提交
4400 4401
    return pInfo->pDelRes;
  }
5
54liuyao 已提交
4402 4403

  if (pInfo->pUpdateRes->info.rows > 0) {
5
54liuyao 已提交
4404 4405
    // process the rest of the data
    pOperator->status = OP_OPENED;
5
54liuyao 已提交
4406
    printDataBlock(pInfo->pUpdateRes, "sems session");
5
54liuyao 已提交
4407 4408
    return pInfo->pUpdateRes;
  }
5
54liuyao 已提交
4409 4410 4411

  pOperator->status = OP_EXEC_DONE;
  return NULL;
5
54liuyao 已提交
4412
}
4413

4414 4415
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                       SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
4416 4417
  int32_t        code = TSDB_CODE_OUT_OF_MEMORY;
  SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo);
4418 4419 4420
  if (pOperator == NULL) {
    goto _error;
  }
4421
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
5
54liuyao 已提交
4422 4423 4424 4425 4426 4427 4428

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
    pInfo->isFinal = true;
    pOperator->name = "StreamSessionFinalAggOperator";
  } else {
    pInfo->isFinal = false;
    pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
5
54liuyao 已提交
4429
    pInfo->pUpdateRes->info.type = STREAM_CLEAR;
5
54liuyao 已提交
4430 4431 4432
    blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
    pOperator->name = "StreamSessionSemiAggOperator";
    pOperator->fpSet =
4433 4434
        createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL,
                            destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
4435 4436 4437 4438 4439
  }
  pOperator->operatorType = pPhyNode->type;
  if (numOfChild > 0) {
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
    for (int32_t i = 0; i < numOfChild; i++) {
4440
      SOperatorInfo* pChild = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
5
54liuyao 已提交
4441 4442 4443 4444
      if (pChild == NULL) {
        goto _error;
      }
      taosArrayPush(pInfo->pChildren, &pChild);
4445 4446 4447 4448 4449 4450
    }
  }
  return pOperator;

_error:
  if (pInfo != NULL) {
4451
    destroyStreamSessionAggOperatorInfo(pInfo, pOperator->exprSupp.numOfExprs);
4452 4453 4454 4455 4456 4457 4458
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5
54liuyao 已提交
4459 4460

void destroyStreamStateOperatorInfo(void* param, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
4461
  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;
4462
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
4463
  destroyStateStreamAggSupporter(&pInfo->streamAggSup);
5
54liuyao 已提交
4464 4465 4466 4467
  cleanupGroupResInfo(&pInfo->groupResInfo);
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
4468
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
5
54liuyao 已提交
4469 4470 4471 4472 4473 4474
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
      destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput);
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
5
54liuyao 已提交
4475 4476 4477 4478
  colDataDestroy(&pInfo->twAggSup.timeWindowData);
  blockDataDestroy(pInfo->pDelRes);
  taosHashCleanup(pInfo->pSeDeleted);
  destroySqlFunctionCtx(pInfo->pDummyCtx, 0);
D
dapan1121 已提交
4479 4480

  taosMemoryFreeClear(param);
5
54liuyao 已提交
4481 4482 4483 4484 4485 4486 4487
}

int64_t getStateWinTsKey(void* data, int32_t index) {
  SStateWindowInfo* pStateWin = taosArrayGet(data, index);
  return pStateWin->winInfo.win.ekey;
}

X
Xiaoyu Wang 已提交
4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499
SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4500 4501 4502 4503 4504 4505 4506 4507
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayPush(pWinInfos, &win);
}

X
Xiaoyu Wang 已提交
4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519
SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayInsert(pWinInfos, index, &win);
}

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {
  if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) {
    return true;
  }
  return false;
}

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {
  return pKeyData && compareVal(pKeyData, &pWin->stateKey);
}

4539 4540 4541
SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
X
Xiaoyu Wang 已提交
4542 4543
  int32_t           size = taosArrayGetSize(pWinInfos);
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index + 1;
      return pWin;
    }
  }
  *pIndex = 0;
  return NULL;
}

4564 4565
SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
                                 SColumn* pCol, int32_t* pIndex) {
4566 4567
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
5
54liuyao 已提交
4568 4569 4570 4571 4572
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    *pIndex = 0;
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
X
Xiaoyu Wang 已提交
4573
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
  *pIndex = index + 1;
  return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol);
}

X
Xiaoyu Wang 已提交
4607 4608
int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, SColumnInfoData* pKeyCol, int32_t rows,
                              int32_t start, bool* allEqual, SHashObj* pSeDelete) {
5
54liuyao 已提交
4609 4610 4611 4612 4613
  *allEqual = true;
  SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex);
  for (int32_t i = start; i < rows; ++i) {
    char* pKeyData = colDataGetData(pKeyCol, i);
    if (!isTsInWindow(pWinInfo, pTs[i])) {
X
Xiaoyu Wang 已提交
4614
      if (isEqualStateKey(pWinInfo, pKeyData)) {
5
54liuyao 已提交
4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628
        int32_t size = taosArrayGetSize(pWinInfos);
        if (winIndex + 1 < size) {
          SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1);
          // ts belongs to the next window
          if (pTs[i] >= pNextWin->winInfo.win.skey) {
            return i - start;
          }
        }
      } else {
        return i - start;
      }
    }
    if (pWinInfo->winInfo.win.skey > pTs[i]) {
      if (pSeDelete && pWinInfo->winInfo.isOutput) {
X
Xiaoyu Wang 已提交
4629 4630
        taosHashPut(pSeDelete, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &pWinInfo->winInfo.win.skey,
                    sizeof(TSKEY));
5
54liuyao 已提交
4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642
        pWinInfo->winInfo.isOutput = false;
      }
      pWinInfo->winInfo.win.skey = pTs[i];
    }
    pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]);
    if (!isEqualStateKey(pWinInfo, pKeyData)) {
      *allEqual = false;
    }
  }
  return rows - start;
}

X
Xiaoyu Wang 已提交
4643 4644
static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int32_t tsIndex, SColumn* pCol,
                                int32_t keyIndex, SHashObj* pSeUpdated, SHashObj* pSeDeleted) {
5
54liuyao 已提交
4645 4646
  SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
  SColumnInfoData* pKeyColInfo = taosArrayGet(pBlock->pDataBlock, keyIndex);
X
Xiaoyu Wang 已提交
4647 4648 4649
  TSKEY*           tsCol = (TSKEY*)pTsColInfo->pData;
  bool             allEqual = false;
  int32_t          step = 1;
5
54liuyao 已提交
4650
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
4651 4652
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
4653
    SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], pBlock->info.groupId, &winIndex);
5
54liuyao 已提交
4654 4655 4656
    if (!pCurWin) {
      continue;
    }
4657
    step = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual,
X
Xiaoyu Wang 已提交
4658
                                 pSeDeleted);
5
54liuyao 已提交
4659 4660
    ASSERT(isTsInWindow(pCurWin, tsCol[i]) || isEqualStateKey(pCurWin, pKeyData));
    taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
5
54liuyao 已提交
4661
    deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
5
54liuyao 已提交
4662 4663 4664
  }
}

X
Xiaoyu Wang 已提交
4665 4666 4667
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated,
                                 SHashObj* pStDeleted) {
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
4668
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4669
  bool                         masterScan = true;
4670
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
X
Xiaoyu Wang 已提交
4671 4672 4673 4674 4675 4676 4677
  int64_t                      groupId = pSDataBlock->info.groupId;
  int64_t                      code = TSDB_CODE_SUCCESS;
  int32_t                      step = 1;
  bool                         ascScan = true;
  TSKEY*                       tsCols = NULL;
  SResultRow*                  pResult = NULL;
  int32_t                      winRows = 0;
5
54liuyao 已提交
4678
  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
4679 4680
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
4681
  } else {
X
Xiaoyu Wang 已提交
4682
    return;
5
54liuyao 已提交
4683
  }
L
Liu Jicong 已提交
4684

5
54liuyao 已提交
4685
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
4686
  blockDataEnsureCapacity(pAggSup->pScanBlock, pSDataBlock->info.rows);
L
Liu Jicong 已提交
4687
  SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
X
Xiaoyu Wang 已提交
4688
  for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
5
54liuyao 已提交
4689
    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
5
54liuyao 已提交
4690 4691 4692
      i++;
      continue;
    }
X
Xiaoyu Wang 已提交
4693 4694 4695
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
    bool              allEqual = true;
4696 4697 4698 4699
    SStateWindowInfo* pCurWin =
        getStateWindow(pAggSup, tsCols[i], pSDataBlock->info.groupId, pKeyData, &pInfo->stateCol, &winIndex);
    winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, pKeyColInfo, pSDataBlock->info.rows, i,
                                    &allEqual, pInfo->pSeDeleted);
5
54liuyao 已提交
4700
    if (!allEqual) {
4701
      appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey,
L
Liu Jicong 已提交
4702
                   &pSDataBlock->info.groupId);
5
54liuyao 已提交
4703
      taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
5
54liuyao 已提交
4704
      deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
5
54liuyao 已提交
4705 4706
      continue;
    }
4707
    code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
4708 4709 4710 4711 4712
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    pCurWin->winInfo.isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
4713
      SWinRes value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId};
4714
      code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &value, sizeof(SWinRes));
5
54liuyao 已提交
4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727
      if (code != TSDB_CODE_SUCCESS) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
      pCurWin->winInfo.isOutput = true;
    }
  }
}

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

4728
  SExprSupp*                   pSup = &pOperator->exprSupp;
5
54liuyao 已提交
4729
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
4730
  SOptrBasicInfo*              pBInfo = &pInfo->binfo;
5
54liuyao 已提交
4731 4732 4733
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4734
      printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4735 4736
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
4737
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
4738
    if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
5
54liuyao 已提交
4739 4740
      doSetOperatorCompleted(pOperator);
    }
5
54liuyao 已提交
4741
    printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4742 4743 4744
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
4745 4746
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
4747
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
4748
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
4749 4750 4751 4752 4753
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
5
54liuyao 已提交
4754
    printDataBlock(pBlock, "single state recv");
4755

5
54liuyao 已提交
4756
    if (pBlock->info.type == STREAM_CLEAR) {
X
Xiaoyu Wang 已提交
4757 4758
      doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
                          pSeUpdated, pInfo->pSeDeleted);
5
54liuyao 已提交
4759
      continue;
4760 4761
    } else if (pBlock->info.type == STREAM_DELETE_DATA) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
5
54liuyao 已提交
4762
      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo);
4763 4764 4765
      copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
      taosArrayDestroy(pWins);
      continue;
4766
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
4767
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState);
5
54liuyao 已提交
4768
      continue;
5
54liuyao 已提交
4769
    }
4770

5
54liuyao 已提交
4771 4772 4773 4774
    if (pInfo->scalarSupp.pExprInfo != NULL) {
      SExprSupp* pExprSup = &pInfo->scalarSupp;
      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
    }
4775
    // the pDataBlock are always the same one, no need to call this again
4776
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
4777 4778 4779 4780 4781
    doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
X
Xiaoyu Wang 已提交
4782

4783
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState,
5
54liuyao 已提交
4784 4785
                     pInfo->ignoreExpiredData, destroyStateWinInfo);
  // closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, destroyStateWinInfo);
5
54liuyao 已提交
4786
  copyUpdateResult(pSeUpdated, pUpdated);
5
54liuyao 已提交
4787 4788
  taosHashCleanup(pSeUpdated);

4789 4790
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
4791 4792 4793 4794
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
5
54liuyao 已提交
4795
    printDataBlock(pInfo->pDelRes, "single state");
5
54liuyao 已提交
4796 4797
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
4798
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
4799
  printDataBlock(pBInfo->pRes, "single state");
5
54liuyao 已提交
4800 4801 4802
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

4803 4804 4805 4806
int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo));
}

X
Xiaoyu Wang 已提交
4807 4808 4809 4810 4811 4812
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
  SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
4813
  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
4814

X
Xiaoyu Wang 已提交
4815 4816
  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
4817 4818 4819 4820
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

4821 4822
  SExprSupp* pSup = &pOperator->exprSupp;

X
Xiaoyu Wang 已提交
4823
  int32_t    numOfCols = 0;
5
54liuyao 已提交
4824 4825 4826
  SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
4827
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5
54liuyao 已提交
4828 4829 4830 4831 4832 4833 4834 4835 4836
  if (pStateNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
  }

4837
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
X
Xiaoyu Wang 已提交
4838 4839
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
5
54liuyao 已提交
4840 4841
      .calTrigger = pStateNode->window.triggerType,
      .maxTs = INT64_MIN,
X
Xiaoyu Wang 已提交
4842
  };
5
54liuyao 已提交
4843
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
4844

4845
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
4846 4847 4848 4849
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

4850
  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
4851 4852 4853 4854 4855 4856 4857 4858 4859
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }

4860
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
4861 4862 4863 4864 4865
  pInfo->primaryTsIndex = tsSlotId;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
4866
  // pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
4867 4868
  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete
5
54liuyao 已提交
4869
  pInfo->pChildren = NULL;
5
54liuyao 已提交
4870
  pInfo->ignoreExpiredData = pStateNode->window.igExpired;
5
54liuyao 已提交
4871 4872

  pOperator->name = "StreamStateAggOperator";
4873
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
5
54liuyao 已提交
4874 4875
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
4876 4877
  pOperator->exprSupp.numOfExprs = numOfCols;
  pOperator->exprSupp.pExprInfo = pExprInfo;
5
54liuyao 已提交
4878 4879
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
4880 4881 4882
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL,
                                         destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType);
5
54liuyao 已提交
4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
  return pOperator;

_error:
  destroyStreamStateOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
4896

4897 4898
void destroyMergeAlignedIntervalOperatorInfo(void* param, int32_t numOfOutput) {
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param;
D
dapan1121 已提交
4899 4900
  destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo, numOfOutput);

D
dapan1121 已提交
4901
  taosMemoryFreeClear(param);
4902 4903
}

4904 4905
static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId,
                                                SSDataBlock* pResultBlock, TSKEY wstartTs) {
4906
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
4907

4908 4909 4910
  SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*            pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*                pSup = &pOperatorInfo->exprSupp;
4911

4912 4913 4914 4915
  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId);
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf,
                                                            GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
  ASSERT(p1 != NULL);
4916

4917 4918
  finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pSup->pCtx, pSup->pExprInfo, pSup->numOfExprs,
                                       pSup->rowEntryInfoOffset, pResultBlock, pTaskInfo);
4919
  taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
4920
  ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0);
4921

4922
  return TSDB_CODE_SUCCESS;
4923 4924
}

4925 4926
static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
                                          SSDataBlock* pBlock, int32_t scanFlag, SSDataBlock* pResultBlock) {
4927
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
D
dapan1121 已提交
4928
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
4929 4930

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
4931
  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
4932 4933

  int32_t     startPos = 0;
4934
  int32_t     numOfOutput = pSup->numOfExprs;
4935
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
4936 4937 4938
  uint64_t    tableGroupId = pBlock->info.groupId;
  SResultRow* pResult = NULL;

4939 4940
  TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols);

4941 4942 4943
  // there is an result exists
  if (miaInfo->curTs != INT64_MIN) {
    ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1);
4944 4945

    if (ts != miaInfo->curTs) {
4946
      outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs);
4947
      miaInfo->curTs = ts;
4948
    }
4949 4950 4951
  } else {
    miaInfo->curTs = ts;
    ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0);
4952 4953 4954
  }

  STimeWindow win = {0};
4955
  win.skey = miaInfo->curTs;
4956 4957
  win.ekey =
      taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1;
4958

4959
  // TODO: remove the hash table (groupid + winkey => result row position)
4960 4961
  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
4962 4963 4964 4965
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
  }

4966 4967
  int32_t currPos = startPos;

4968
  STimeWindow currWin = win;
4969
  while (++currPos < pBlock->info.rows) {
4970
    if (tsCols[currPos] == miaInfo->curTs) {
4971
      continue;
4972 4973 4974 4975 4976 4977
    }

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
    doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
                     tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder);

4978 4979
    outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs);
    miaInfo->curTs = tsCols[currPos];
4980

4981
    currWin.skey = miaInfo->curTs;
4982
    currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit,
4983 4984
                               iaInfo->interval.precision) -
                   1;
4985 4986 4987 4988 4989 4990

    startPos = currPos;
    ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
4991
    }
4992 4993

    miaInfo->curTs = currWin.skey;
4994
  }
4995

4996
  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
4997
  doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
4998
                   tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder);
4999 5000
}

5001 5002

static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
S
shenglian zhou 已提交
5003
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
5004

5005
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperator->info;
D
dapan1121 已提交
5006
  SIntervalAggOperatorInfo*             iaInfo = miaInfo->intervalAggOperatorInfo;
5007

5008
  SExprSupp*   pSup = &pOperator->exprSupp;
5009
  SSDataBlock* pRes = iaInfo->binfo.pRes;
5010

5011 5012
  SOperatorInfo* downstream = pOperator->pDownstream[0];
  int32_t        scanFlag = MAIN_SCAN;
5013

5014 5015 5016 5017 5018 5019 5020
  while (1) {
    SSDataBlock* pBlock = NULL;
    if (miaInfo->prefetchedBlock == NULL) {
      pBlock = downstream->fpSet.getNextFn(downstream);
    } else {
      pBlock = miaInfo->prefetchedBlock;
      miaInfo->prefetchedBlock = NULL;
5021

5022 5023
      miaInfo->groupId = pBlock->info.groupId;
    }
5024

5025 5026 5027 5028
    if (pBlock == NULL) {
      // close last unfinalized time window
      if (miaInfo->curTs != INT64_MIN) {
        ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1);
5029 5030
        outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs);
        miaInfo->curTs = INT64_MIN;
5031
      }
5032

5033 5034
      doSetOperatorCompleted(pOperator);
      break;
5035
    }
5036

5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047
    if (!miaInfo->hasGroupId) {
      miaInfo->hasGroupId = true;
      miaInfo->groupId = pBlock->info.groupId;
    } else if (miaInfo->groupId != pBlock->info.groupId) {
      // if there are unclosed time window, close it firstly.
      ASSERT(miaInfo->curTs != INT64_MIN);
      outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs);
      miaInfo->prefetchedBlock = pBlock;
      miaInfo->curTs = INT64_MIN;
      break;
    }
5048

5049 5050 5051 5052 5053 5054 5055 5056
    getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag);
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true);
    doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);

    doFilter(miaInfo->pCondition, pRes, NULL);
    if (pRes->info.rows >= pOperator->resultInfo.capacity) {
      break;
    }
5057 5058
  }

5059
  pRes->info.groupId = miaInfo->groupId;
5060
  miaInfo->hasGroupId = false;
5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077
}

static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info;
  SIntervalAggOperatorInfo*             iaInfo = pMiaInfo->intervalAggOperatorInfo;
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);

  if (iaInfo->binfo.mergeResultBlock) {
    while(1) {
      if (pOperator->status == OP_EXEC_DONE) {
5078 5079
        break;
      }
5080

5081
      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
5082 5083 5084
        break;
      }

5085 5086 5087 5088
      doMergeAlignedIntervalAgg(pOperator);
    }
  } else {
    doMergeAlignedIntervalAgg(pOperator);
5089 5090 5091 5092 5093 5094 5095
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

5096 5097
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo,
                                                      int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval,
5098
                                                      int32_t primaryTsSlotId, SNode* pCondition, bool mergeResultBlock,
5099
                                                      SExecTaskInfo* pTaskInfo) {
5100
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo));
5101
  SOperatorInfo*                        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5102 5103 5104 5105
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

D
dapan1121 已提交
5106 5107 5108 5109 5110 5111
  miaInfo->intervalAggOperatorInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
  if (miaInfo->intervalAggOperatorInfo == NULL) {
    goto _error;
  }

  SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo;
5112
  SExprSupp*                pSup = &pOperator->exprSupp;
5113

5114
  miaInfo->pCondition = pCondition;
5115 5116
  miaInfo->curTs = INT64_MIN;

5117
  iaInfo->win = pTaskInfo->window;
5118
  iaInfo->inputOrder = TSDB_ORDER_ASC;
5119
  iaInfo->interval = *pInterval;
5120 5121
  iaInfo->execModel = pTaskInfo->execModel;
  iaInfo->primaryTsIndex = primaryTsSlotId;
5122
  iaInfo->binfo.mergeResultBlock = mergeResultBlock;
5123 5124

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
5125
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5126

5127 5128
  int32_t code =
      initAggInfo(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
5129
  initBasicInfo(&iaInfo->binfo, pResBlock);
5130

5131
  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
5132

5133
  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, iaInfo);
5134 5135
  if (iaInfo->timeWindowInterpo) {
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
5136 5137
  }

5138
  if (code != TSDB_CODE_SUCCESS) {
5139 5140 5141
    goto _error;
  }

5142
  initResultRowInfo(&iaInfo->binfo.resultRowInfo);
5143
  blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity);
5144

5145 5146
  pOperator->name = "TimeMergeAlignedIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL;
5147 5148
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
5149
  pOperator->exprSupp.pExprInfo = pExprInfo;
5150
  pOperator->pTaskInfo = pTaskInfo;
5151
  pOperator->exprSupp.numOfExprs = numOfCols;
5152
  pOperator->info = miaInfo;
5153

5154
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL,
5155
                                         destroyMergeAlignedIntervalOperatorInfo, NULL, NULL, NULL);
5156 5157 5158 5159 5160 5161 5162 5163 5164

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
5165
  destroyMergeAlignedIntervalOperatorInfo(miaInfo, numOfCols);
5166
  taosMemoryFreeClear(miaInfo);
5167 5168 5169 5170
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5171 5172 5173 5174 5175

//=====================================================================================================================
// merge interval operator
typedef struct SMergeIntervalAggOperatorInfo {
  SIntervalAggOperatorInfo intervalAggOperatorInfo;
L
Liu Jicong 已提交
5176 5177 5178 5179 5180 5181
  SList*                   groupIntervals;
  SListIter                groupIntervalsIter;
  bool                     hasGroupId;
  uint64_t                 groupId;
  SSDataBlock*             prefetchedBlock;
  bool                     inputBlocksFinished;
5182 5183
} SMergeIntervalAggOperatorInfo;

S
slzhou 已提交
5184
typedef struct SGroupTimeWindow {
L
Liu Jicong 已提交
5185
  uint64_t    groupId;
S
slzhou 已提交
5186 5187 5188
  STimeWindow window;
} SGroupTimeWindow;

5189 5190
void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) {
  SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
S
slzhou 已提交
5191
  tdListFree(miaInfo->groupIntervals);
5192
  destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput);
5193

D
dapan1121 已提交
5194
  taosMemoryFreeClear(param);
5195 5196
}

L
Liu Jicong 已提交
5197 5198
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win,
                                    SSDataBlock* pResultBlock) {
5199 5200 5201
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
5202
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId);
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf,
                                                            GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
  ASSERT(p1 != NULL);
  finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo,
                                       pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo);
  taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
  return TSDB_CODE_SUCCESS;
}

5215 5216 5217 5218 5219
static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock,
                                        STimeWindow* newWin) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
5220
  bool                           ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5221 5222
  SExprSupp*                     pExprSup = &pOperatorInfo->exprSupp;

S
slzhou 已提交
5223 5224
  SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin};
  tdListAppend(miaInfo->groupIntervals, &groupTimeWindow);
5225

S
slzhou 已提交
5226 5227 5228 5229 5230
  SListIter iter = {0};
  tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD);
  SListNode* listNode = NULL;
  while ((listNode = tdListNext(&iter)) != NULL) {
    SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
L
Liu Jicong 已提交
5231
    if (prevGrpWin->groupId != tableGroupId) {
S
slzhou 已提交
5232 5233 5234
      continue;
    }
    STimeWindow* prevWin = &prevGrpWin->window;
H
Haojun Liao 已提交
5235
    if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) {
S
slzhou 已提交
5236 5237 5238
      finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
      tdListPopNode(miaInfo->groupIntervals, listNode);
    }
5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255
  }

  return 0;
}

static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                                   int32_t scanFlag, SSDataBlock* pResultBlock) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
  SExprSupp*     pExprSup = &pOperatorInfo->exprSupp;

  int32_t     startPos = 0;
  int32_t     numOfOutput = pExprSup->numOfExprs;
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
5256
  bool        ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC);
5257 5258 5259
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

5260 5261
  STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval,
                                        iaInfo->inputOrder);
5262 5263 5264 5265 5266 5267 5268 5269 5270 5271

  int32_t ret =
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                             numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
  }

  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
5272
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292
  ASSERT(forwardRows > 0);

  // prev time window not interpolation yet.
  if (iaInfo->timeWindowInterpo) {
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult);
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);

    // restore current time window
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
                                 numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    // window start key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup);
  }

  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true);
  doApplyFunctions(pTaskInfo, pExprSup->pCtx, &win, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
5293
                   pBlock->info.rows, numOfOutput, iaInfo->inputOrder);
5294 5295 5296 5297 5298 5299 5300 5301
  doCloseWindow(pResultRowInfo, iaInfo, pResult);

  // output previous interval results after this interval (&win) is closed
  outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win);

  STimeWindow nextWin = win;
  while (1) {
    int32_t prevEndPos = forwardRows - 1 + startPos;
5302 5303
    startPos =
        getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->inputOrder);
5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
    int32_t code =
        setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                               pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    ekey = ascScan ? nextWin.ekey : nextWin.skey;
    forwardRows =
5318
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder);
5319 5320 5321 5322 5323 5324

    // window start(end) key interpolation
    doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup);

    updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true);
    doApplyFunctions(pTaskInfo, pExprSup->pCtx, &nextWin, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
5325
                     tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder);
5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365
    doCloseWindow(pResultRowInfo, iaInfo, pResult);

    // output previous interval results after this interval (&nextWin) is closed
    outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin);
  }

  if (iaInfo->timeWindowInterpo) {
    saveDataBlockLastRow(iaInfo->pPrevValues, pBlock, iaInfo->pInterpCols);
  }
}

static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

  SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExprSupp*                     pExpSupp = &pOperator->exprSupp;

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pRes = iaInfo->binfo.pRes;
  blockDataCleanup(pRes);
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);

  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
        miaInfo->groupId = pBlock->info.groupId;
        miaInfo->prefetchedBlock = NULL;
      }

      if (pBlock == NULL) {
S
slzhou 已提交
5366
        tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD);
5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378
        miaInfo->inputBlocksFinished = true;
        break;
      }

      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
        miaInfo->groupId = pBlock->info.groupId;
      } else if (miaInfo->groupId != pBlock->info.groupId) {
        miaInfo->prefetchedBlock = pBlock;
        break;
      }

5379 5380
      getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag);
      setInputDataBlock(pOperator, pExpSupp->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true);
5381 5382 5383 5384 5385 5386 5387 5388
      doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);

      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
        break;
      }
    }

    pRes->info.groupId = miaInfo->groupId;
5389 5390 5391
  }

  if (miaInfo->inputBlocksFinished) {
S
slzhou 已提交
5392
    SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter);
5393

S
slzhou 已提交
5394 5395 5396 5397
    if (listNode != NULL) {
      SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data);
      finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
      pRes->info.groupId = grpWin->groupId;
5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411
    }
  }

  if (pRes->info.rows == 0) {
    doSetOperatorCompleted(pOperator);
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                               SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
5412
                                               bool mergeBlock, SExecTaskInfo* pTaskInfo) {
5413 5414 5415 5416 5417 5418
  SMergeIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

S
slzhou 已提交
5419
  miaInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow));
5420

5421 5422
  SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
  iaInfo->win = pTaskInfo->window;
5423
  iaInfo->inputOrder = TSDB_ORDER_ASC;
5424 5425
  iaInfo->interval = *pInterval;
  iaInfo->execModel = pTaskInfo->execModel;
5426
  iaInfo->binfo.mergeResultBlock = mergeBlock;
5427 5428 5429 5430 5431 5432

  iaInfo->primaryTsIndex = primaryTsSlotId;

  SExprSupp* pExprSupp = &pOperator->exprSupp;

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
5433
  initResultSizeInfo(&pOperator->resultInfo, 4096);
5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474

  int32_t code = initAggInfo(pExprSupp, &iaInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&iaInfo->binfo, pResBlock);

  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);

  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, numOfCols, iaInfo);
  if (iaInfo->timeWindowInterpo) {
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
    if (iaInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
  }

  initResultRowInfo(&iaInfo->binfo.resultRowInfo);

  pOperator->name = "TimeMergeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL;
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->exprSupp.pExprInfo = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
  pOperator->info = miaInfo;

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL,
                                         destroyMergeIntervalOperatorInfo, NULL, NULL, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
  destroyMergeIntervalOperatorInfo(miaInfo, numOfCols);
  taosMemoryFreeClear(miaInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
5475
}