timewindowoperator.c 138.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15
#include "executorimpl.h"
X
Xiaoyu Wang 已提交
16
#include "function.h"
5
54liuyao 已提交
17
#include "functionMgt.h"
L
Liu Jicong 已提交
18 19
#include "tdatablock.h"
#include "ttime.h"
H
Haojun Liao 已提交
20
#include "tfill.h"
21 22 23 24 25 26

typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;

5
54liuyao 已提交
27
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
5
54liuyao 已提交
28

29 30 31 32 33
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);

static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult);
static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult);

H
Haojun Liao 已提交
34 35 36 37 38 39 40 41 42 43 44 45
///*
// * There are two cases to handle:
// *
// * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including
// * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey.
// * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be
// *    merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there
// *    is a previous result generated or not.
// */
//static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) {
//  // do nothing
//}
46

X
Xiaoyu Wang 已提交
47
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69

static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w,
                                      bool ascQuery) {
  if (ascQuery) {
    getAlignQueryTimeWindow(pInterval, precision, ts, w);
  } else {
    // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
    getAlignQueryTimeWindow(pInterval, precision, ts, w);

    int64_t key = w->skey;
    while (key < ts) {  // moving towards end
      key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision);
      if (key >= ts) {
        break;
      }

      w->skey = key;
    }
  }
}

// get the correct time window according to the handled timestamp
X
Xiaoyu Wang 已提交
70 71
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
                                int32_t precision, STimeWindow* win) {
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
  STimeWindow w = {0};

  if (pResultRowInfo->cur.pageId == -1) {  // the first window, from the previous stored value
    getInitialStartTimeWindow(pInterval, precision, ts, &w, true);
    w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
  } else {
    w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win;
  }

  if (w.skey > ts || w.ekey < ts) {
    if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
      w.skey = taosTimeTruncate(ts, pInterval, precision);
      w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
    } else {
      int64_t st = w.skey;

      if (st > ts) {
        st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
      }

      int64_t et = st + pInterval->interval - 1;
      if (et < ts) {
        st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
      }

      w.skey = st;
      w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
    }
  }
  return w;
}

static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
                                      int32_t numOfOutput, int32_t* rowCellInfoOffset, SAggSupporter* pAggSup,
                                      SExecTaskInfo* pTaskInfo) {
  assert(win->skey <= win->ekey);
  SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                  masterscan, tableGroupId, pTaskInfo, true, pAggSup);

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // set time window for current result
  pResultRow->win = (*win);
119

120
  *pResult = pResultRow;
121
  setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowCellInfoOffset);
122

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
  return TSDB_CODE_SUCCESS;
}

static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) {
  int64_t* ts = (int64_t*)pColData->pData;
  int32_t  delta = includeEndpoint ? 1 : 0;

  int64_t duration = pWin->ekey - pWin->skey + delta;
  ts[2] = duration;            // set the duration
  ts[3] = pWin->skey;          // window start key
  ts[4] = pWin->ekey + delta;  // window end key
}

static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) {
  pRowSup->win.ekey = ts;
  pRowSup->prevTs = ts;
  pRowSup->numOfRows += 1;
}

static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex) {
  pRowSup->startRowIndex = rowIndex;
  pRowSup->numOfRows = 0;
  pRowSup->win.skey = tsList[rowIndex];
}

static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
                                                   int16_t pos, int16_t order, int64_t* pData) {
150
  int32_t forwardRows = 0;
151 152 153 154

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
155
      forwardRows = end;
156 157

      if (pData[end + pos] == ekey) {
158
        forwardRows += 1;
159 160 161
      }
    }
  } else {
162
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
163
    if (end >= 0) {
164
      forwardRows = end;
165

166
      if (pData[end + pos] == ekey) {
167
        forwardRows += 1;
168 169
      }
    }
X
Xiaoyu Wang 已提交
170 171 172 173 174 175 176 177
    //    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
    //    if (end >= 0) {
    //      forwardRows = pos - end;
    //
    //      if (pData[end] == ekey) {
    //        forwardRows += 1;
    //      }
    //    }
178 179
  }

180 181
  assert(forwardRows >= 0);
  return forwardRows;
182 183
}

5
54liuyao 已提交
184
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY*  keyList = (TSKEY*)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
201 202 203 204 205 206 207 208 209 210 211
      if (key >= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key < keyList[lastPos]) {
        lastPos += 1;
        if (lastPos >= num) {
          return -1;
        } else {
          return lastPos;
        }
      }
212 213 214 215 216 217

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        firstPos = midPos + 1;
218 219
      } else if (key > keyList[midPos]) {
        lastPos = midPos - 1;
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

X
Xiaoyu Wang 已提交
255 256
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
                                 __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num = -1;
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);

  if (order == TSDB_ORDER_ASC) {
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
      if (item != NULL) {
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
278
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
279 280
      }
    } else {
281
      num = pDataBlockInfo->rows - startPos;
282
      if (item != NULL) {
283
        item->lastKey = pDataBlockInfo->window.ekey + step;
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
      }
    }
  }

  assert(num >= 0);
  return num;
}

static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
    return;
  }

  int64_t key = tw->skey, interval = pInterval->interval;
  // convert key to second
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;

  if (pInterval->intervalUnit == 'y') {
    interval *= 12;
  }

  struct tm tm;
  time_t    t = (time_t)key;
  taosLocalTime(&t, &tm);

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision);

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision);

  tw->ekey -= 1;
}

X
Xiaoyu Wang 已提交
325
void doTimeWindowInterpolation(SIntervalAggOperatorInfo* pInfo, int32_t numOfExprs, SArray* pDataBlock, TSKEY prevTs,
326
                               int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) {
327
  SqlFunctionCtx* pCtx = pInfo->binfo.pCtx;
328

329 330
  int32_t index = 1;
  for (int32_t k = 0; k < numOfExprs; ++k) {
H
Haojun Liao 已提交
331
    if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
332 333 334 335
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

X
Xiaoyu Wang 已提交
336
    SFunctParam*     pParam = &pCtx[k].param[0];
337 338
    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId);

339
    ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey);
340

341
    double v1 = 0, v2 = 0, v = 0;
342
    if (prevRowIndex == -1) {
343 344
      SGroupKeys* p = taosArrayGet(pInfo->pPrevValues, index);
      GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData);
345
    } else {
346
      GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex));
347 348
    }

349
    GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
350

351
#if 0
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            //            pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
371 372
#endif

X
Xiaoyu Wang 已提交
373 374 375
    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
    SPoint point = (SPoint){.key = windowKey, .val = &v};
376

X
Xiaoyu Wang 已提交
377
    taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
378

X
Xiaoyu Wang 已提交
379 380 381 382 383 384
    if (type == RESULT_ROW_START_INTERP) {
      pCtx[k].start.key = point.key;
      pCtx[k].start.val = v;
    } else {
      pCtx[k].end.key = point.key;
      pCtx[k].end.val = v;
385
    }
X
Xiaoyu Wang 已提交
386 387 388

    index += 1;
  }
389
#if 0
390
  }
391
#endif
392 393 394 395 396 397 398 399 400 401 402 403 404 405
}

static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

X
Xiaoyu Wang 已提交
406 407 408
static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs,
                                              int32_t pos, SSDataBlock* pBlock, const TSKEY* tsCols, STimeWindow* win) {
  bool ascQuery = (pInfo->order == TSDB_ORDER_ASC);
409

410
  TSKEY curTs = tsCols[pos];
411 412

  SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
X
Xiaoyu Wang 已提交
413
  TSKEY       lastTs = *(int64_t*)pTsKey->pData;
414 415 416 417 418

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery ? win->skey : win->ekey;
  if (key == curTs) {
419
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP);
420 421 422
    return true;
  }

423 424
  // it is the first time window, no need to do interpolation
  if (pTsKey->isNull && pos == 0) {
425
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP);
426 427 428 429
  } else {
    TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]);
    doTimeWindowInterpolation(pInfo, numOfExprs, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key,
                              RESULT_ROW_START_INTERP);
430 431 432 433 434
  }

  return true;
}

X
Xiaoyu Wang 已提交
435 436 437
static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs,
                                            int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols,
                                            TSKEY blockEkey, STimeWindow* win) {
438
  int32_t order = pInfo->order;
439 440

  TSKEY actualEndKey = tsCols[endRowIndex];
441
  TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey;
442 443

  // not ended in current data block, do not invoke interpolation
444 445
  if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) {
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP);
446 447 448
    return false;
  }

449
  // there is actual end point of current time window, no interpolation needs
450
  if (key == actualEndKey) {
451
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP);
452 453 454
    return true;
  }

455
  int32_t nextRowIndex = endRowIndex + 1;
456 457 458
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
X
Xiaoyu Wang 已提交
459 460
  doTimeWindowInterpolation(pInfo, numOfExprs, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key,
                            RESULT_ROW_END_INTERP);
461 462 463 464
  return true;
}

static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
5
54liuyao 已提交
465
                                      TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
X
Xiaoyu Wang 已提交
466
  bool ascQuery = (order == TSDB_ORDER_ASC);
467 468 469 470 471 472 473 474 475 476

  int32_t precision = pInterval->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
    return -1;
  }

477
  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
478 479 480 481
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
482
    startPos = prevPosition + 1;
483
  } else {
484
    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
485 486
      startPos = 0;
    } else {
487
      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
    }
  }

  /* interp query with fill should not skip time window */
  //  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
  //    return startPos;
  //  }

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
    if (ascQuery) {
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
      }
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
      }
    }
  }

  return startPos;
}

531 532
static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
  ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp == true;
  }
}

static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp = true;
  }
}

X
Xiaoyu Wang 已提交
549 550 551
static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t numOfExprs,
                                        SqlFunctionCtx* pCtx, SResultRow* pResult, STimeWindow* win, int32_t startPos,
                                        int32_t forwardRows) {
552
  if (!pInfo->timeWindowInterpo) {
553 554 555
    return;
  }

556
  ASSERT(pBlock != NULL);
557 558 559 560 561
  if (pBlock->pDataBlock == NULL) {
    //    tscError("pBlock->pDataBlock == NULL");
    return;
  }

562
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
563 564

  TSKEY* tsCols = (TSKEY*)(pColInfo->pData);
565
  bool   done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
566
  if (!done) {  // it is not interpolated, now start to generated the interpolated value
567
    bool interp = setTimeWindowInterpolationStartTs(pInfo, pCtx, numOfExprs, startPos, pBlock, tsCols, win);
568 569 570 571
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
572
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP);
573 574 575 576 577 578 579 580
  }

  // point interpolation does not require the end key time window interpolation.
  //  if (pointInterpQuery) {
  //    return;
  //  }

  // interpolation query does not generate the time window end interpolation
581
  done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
582
  if (!done) {
583
    int32_t endRowIndex = startPos + forwardRows - 1;
584

585
    TSKEY endKey = (pInfo->order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
586
    bool  interp =
587
        setTimeWindowInterpolationEndTs(pInfo, pCtx, numOfExprs, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
588 589 590 591
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
592
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP);
593 594 595
  }
}

596 597
static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) {
  if (pBlock->pDataBlock == NULL) {
598 599 600
    return;
  }

601 602 603 604 605 606 607
  size_t num = taosArrayGetSize(pPrevKeys);
  for (int32_t k = 0; k < num; ++k) {
    SColumn* pc = taosArrayGet(pCols, k);

    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId);

    SGroupKeys* pkey = taosArrayGet(pPrevKeys, k);
X
Xiaoyu Wang 已提交
608
    for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) {
609 610 611 612 613 614 615 616 617 618 619 620 621 622
      if (colDataIsNull_s(pColInfo, i)) {
        continue;
      }

      char* val = colDataGetData(pColInfo, i);
      if (IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, varDataTLen(val));
        ASSERT(varDataTLen(val) <= pkey->bytes);
      } else {
        memcpy(pkey->pData, val, pkey->bytes);
      }

      break;
    }
623 624 625
  }
}

626 627 628 629
static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo,
                                       SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) {
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

630
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
631

632 633 634
  int32_t  startPos = 0;
  int32_t  numOfOutput = pOperatorInfo->numOfExprs;
  uint64_t groupId = pBlock->info.groupId;
635

636
  SResultRow* pResult = NULL;
637

638 639
  while (1) {
    SListNode* pn = tdListGetHead(pResultRowInfo->openWindow);
640

641 642 643 644
    SResultRowPosition* p1 = (SResultRowPosition*)pn->data;
    if (p->pageId == p1->pageId && p->offset == p1->offset) {
      break;
    }
645

646 647
    SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1);
    ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
648

649
    if (pr->closed) {
X
Xiaoyu Wang 已提交
650 651
      ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
             isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
652 653 654
      tdListPopHead(pResultRowInfo->openWindow);
      continue;
    }
655

656
    STimeWindow w = pr->win;
X
Xiaoyu Wang 已提交
657 658 659
    int32_t     ret =
        setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pInfo->binfo.pCtx,
                               numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
660 661 662 663 664 665
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP));

X
Xiaoyu Wang 已提交
666 667 668 669
    SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
    int64_t     prevTs = *(int64_t*)pTsKey->pData;
    doTimeWindowInterpolation(pInfo, numOfOutput, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey,
                              RESULT_ROW_END_INTERP);
670 671 672 673 674 675 676 677 678 679

    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    setNotInterpoWindowKey(pInfo->binfo.pCtx, numOfExprs, RESULT_ROW_START_INTERP);

    doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &w, &pInfo->twAggSup.timeWindowData, startPos, 0, tsCols,
                     pBlock->info.rows, numOfExprs, pInfo->order);

    if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
      closeResultRow(pr);
      tdListPopHead(pResultRowInfo->openWindow);
X
Xiaoyu Wang 已提交
680
    } else {  // the remains are can not be closed yet.
681
      break;
682
    }
683
  }
684
}
685

5
54liuyao 已提交
686
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
687

X
Xiaoyu Wang 已提交
688 689 690
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;
5
54liuyao 已提交
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (key >= getValuefn(keyList, lastPos)) return lastPos;
      if (key == getValuefn(keyList, firstPos)) return firstPos;
      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (key <= getValuefn(keyList, firstPos)) return firstPos;
      if (key == getValuefn(keyList, lastPos)) return lastPos;

      if (key > getValuefn(keyList, lastPos)) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
737 738
  }

5
54liuyao 已提交
739 740 741 742
  return midPos;
}

int64_t getReskey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
743
  SArray*     res = (SArray*)data;
5
54liuyao 已提交
744 745 746 747
  SResKeyPos* pos = taosArrayGetP(res, index);
  return *(int64_t*)pos->key;
}

5
54liuyao 已提交
748 749
static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId,
    SArray* pUpdated) {
5
54liuyao 已提交
750
  int32_t size = taosArrayGetSize(pUpdated);
5
54liuyao 已提交
751
  int32_t index = binarySearch(pUpdated, size, ts, TSDB_ORDER_DESC, getReskey);
5
54liuyao 已提交
752 753 754 755
  if (index == -1) {
    index = 0;
  } else {
    TSKEY resTs = getReskey(pUpdated, index);
5
54liuyao 已提交
756
    if (resTs < ts) {
5
54liuyao 已提交
757 758 759 760 761
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
H
Haojun Liao 已提交
762

5
54liuyao 已提交
763 764 765 766 767
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
5
54liuyao 已提交
768 769
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
X
Xiaoyu Wang 已提交
770
  if (taosArrayInsert(pUpdated, index, &newPos) == NULL) {
5
54liuyao 已提交
771 772 773 774 775
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
776 777 778 779
static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
  return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated);
}

5
54liuyao 已提交
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795
static void removeResult(SArray* pUpdated, TSKEY key) {
  int32_t size = taosArrayGetSize(pUpdated);
  int32_t index = binarySearch(pUpdated, size, key, TSDB_ORDER_DESC, getReskey);
  if (index >= 0 && key == getReskey(pUpdated, index)) {
    taosArrayRemove(pUpdated, index);
  }
}

static void removeResults(SArray* pWins, SArray* pUpdated) {
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
    STimeWindow* pW = taosArrayGet(pWins, i);
    removeResult(pUpdated, pW->skey);
  }
}

5
54liuyao 已提交
796
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
797
                            int32_t scanFlag, SArray* pUpdated) {
798
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
799

800 801
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

X
Xiaoyu Wang 已提交
802 803 804 805 806 807 808
  int32_t     startPos = 0;
  int32_t     numOfOutput = pOperatorInfo->numOfExprs;
  int64_t*    tsCols = extractTsCol(pBlock, pInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
  bool        ascScan = (pInfo->order == TSDB_ORDER_ASC);
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;
809 810 811 812

  STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
                                        pInfo->interval.precision, &pInfo->win);

X
Xiaoyu Wang 已提交
813 814 815
  int32_t ret =
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx,
                             numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
816 817 818 819
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
  }

5
54liuyao 已提交
820
  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
821
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
822
      saveResultRow(pResult, tableGroupId, pUpdated);
5
54liuyao 已提交
823
    }
824 825
  }

X
Xiaoyu Wang 已提交
826 827 828
  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
829
  ASSERT(forwardRows > 0);
830 831

  // prev time window not interpolation yet.
832
  if (pInfo->timeWindowInterpo) {
833 834
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult);
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);
835 836

    // restore current time window
X
Xiaoyu Wang 已提交
837 838 839
    ret =
        setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx,
                               numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
840 841 842 843
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

844
    // window start key interpolation
845
    doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &win, startPos, forwardRows);
846
  }
847 848

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
849 850 851 852
  doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
                   pBlock->info.rows, numOfOutput, pInfo->order);

  doCloseWindow(pResultRowInfo, pInfo, pResult);
853 854 855

  STimeWindow nextWin = win;
  while (1) {
856
    int32_t prevEndPos = forwardRows - 1 + startPos;
857
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->order);
858 859 860 861 862
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
X
Xiaoyu Wang 已提交
863 864 865
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                          pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset,
                                          &pInfo->aggSup, pTaskInfo);
866 867 868 869
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

5
54liuyao 已提交
870
    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
871
      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
872
        saveResultRow(pResult, tableGroupId, pUpdated);
5
54liuyao 已提交
873
      }
874 875
    }

X
Xiaoyu Wang 已提交
876
    ekey = ascScan ? nextWin.ekey : nextWin.skey;
877
    forwardRows =
878
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
879 880

    // window start(end) key interpolation
X
Xiaoyu Wang 已提交
881 882
    doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos,
                                forwardRows);
883 884

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
X
Xiaoyu Wang 已提交
885 886
    doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     tsCols, pBlock->info.rows, numOfOutput, pInfo->order);
887
    doCloseWindow(pResultRowInfo, pInfo, pResult);
888 889 890
  }

  if (pInfo->timeWindowInterpo) {
891
    saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
892
  }
893 894 895 896 897 898 899 900 901 902 903 904
}

void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
  // current result is done in computing final results.
  if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
    closeResultRow(pResult);
    tdListPopHead(pResultRowInfo->openWindow);
  }
}

SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult) {
  SResultRowPosition pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
X
Xiaoyu Wang 已提交
905
  SListNode*         pn = tdListGetTail(pResultRowInfo->openWindow);
906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930
  if (pn == NULL) {
    tdListAppend(pResultRowInfo->openWindow, &pos);
    return pos;
  }

  SResultRowPosition* px = (SResultRowPosition*)pn->data;
  if (px->pageId != pos.pageId || px->offset != pos.offset) {
    tdListAppend(pResultRowInfo->openWindow, &pos);
  }

  return pos;
}

int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
  TSKEY* tsCols = NULL;
  if (pBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;

    if (tsCols != NULL) {
      blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
    }
  }

  return tsCols;
931 932 933 934 935 936 937
}

static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
  }

L
Liu Jicong 已提交
938
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
939
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
940

941 942
  int32_t scanFlag = MAIN_SCAN;

X
Xiaoyu Wang 已提交
943
  int64_t        st = taosGetTimestampUs();
944 945 946
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
947
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
948 949 950 951
    if (pBlock == NULL) {
      break;
    }

952 953
    getTableScanInfo(pOperator, &pInfo->order, &scanFlag);

954
    // the pDataBlock are always the same one, no need to call this again
955
    setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, scanFlag, true);
H
Haojun Liao 已提交
956
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL);
957 958

#if 0  // test for encode/decode result info
959
    if(pOperator->fpSet.encodeResultRow){
960 961 962
      char *result = NULL;
      int32_t length = 0;
      SAggSupporter   *pSup = &pInfo->aggSup;
963
      pOperator->fpSet.encodeResultRow(pOperator, &result, &length);
964 965
      taosHashClear(pSup->pResultRowHashTable);
      pInfo->binfo.resultRowInfo.size = 0;
966
      pOperator->fpSet.decodeResultRow(pOperator, result);
967 968 969 970 971 972 973 974
      if(result){
        taosMemoryFree(result);
      }
    }
#endif
  }

  closeAllResultRows(&pInfo->binfo.resultRowInfo);
975
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->order);
976
  OPTR_SET_OPENED(pOperator);
977 978

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
979 980 981
  return TSDB_CODE_SUCCESS;
}

982 983 984 985 986 987 988 989 990 991 992 993
static bool compareVal(const char* v, const SStateKeys* pKey) {
  if (IS_VAR_DATA_TYPE(pKey->type)) {
    if (varDataLen(v) != varDataLen(pKey->pData)) {
      return false;
    } else {
      return strncmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0;
    }
  } else {
    return memcmp(pKey->pData, v, pKey->bytes) == 0;
  }
}

994
static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
L
Liu Jicong 已提交
995
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
996

997
  SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
998 999 1000
  int64_t          gid = pBlock->info.groupId;

  bool    masterScan = true;
1001
  int32_t numOfOutput = pOperator->numOfExprs;
1002 1003
  int16_t bytes = pStateColInfoData->info.bytes;

1004
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1005 1006 1007 1008 1009
  TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

1010
  struct SColumnDataAgg* pAgg = NULL;
1011
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
X
Xiaoyu Wang 已提交
1012
    pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
1013
    if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
1014 1015 1016 1017 1018 1019
      continue;
    }

    char* val = colDataGetData(pStateColInfoData, j);

    if (!pInfo->hasKey) {
1020 1021 1022 1023 1024 1025 1026
      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }

1027 1028 1029 1030
      pInfo->hasKey = true;

      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
1031
    } else if (compareVal(val, &pInfo->stateKey)) {
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
      doKeepTuple(pRowSup, tsList[j]);
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // a new state window started
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
      int32_t ret =
          setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pInfo->binfo.pCtx,
                                 numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
      }

      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1051
      doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1052 1053 1054 1055 1056
                       pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);

      // here we start a new session window
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
1057 1058 1059 1060 1061 1062 1063

      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
  int32_t ret =
      setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pInfo->binfo.pCtx,
                             numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
1077
  doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1078 1079 1080
                   pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
}

1081
static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
1082 1083 1084 1085 1086
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
1087 1088 1089

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
  SOptrBasicInfo* pBInfo = &pInfo->binfo;
1090 1091

  if (pOperator->status == OP_RES_TO_RETURN) {
1092
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1093
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1094 1095 1096 1097 1098 1099 1100
      doSetOperatorCompleted(pOperator);
      return NULL;
    }

    return pBInfo->pRes;
  }

1101
  int32_t order = TSDB_ORDER_ASC;
1102
  int64_t st = taosGetTimestampUs();
1103 1104 1105

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  while (1) {
1106
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1107 1108 1109 1110
    if (pBlock == NULL) {
      break;
    }

1111
    setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, MAIN_SCAN, true);
1112 1113
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1114 1115 1116
    doStateWindowAggImpl(pOperator, pInfo, pBlock);
  }

X
Xiaoyu Wang 已提交
1117
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1118

1119 1120 1121
  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pBInfo->resultRowInfo);

1122
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1123
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1124
  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1125
  if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1126 1127 1128
    doSetOperatorCompleted(pOperator);
  }

1129 1130 1131
  size_t rows = pBInfo->pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1132
  return (rows == 0) ? NULL : pBInfo->pRes;
1133 1134
}

1135
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
1136
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
1137
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1138 1139 1140 1141 1142 1143 1144 1145

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pBlock = pInfo->binfo.pRes;

  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
1146
    return pOperator->fpSet.getStreamResFn(pOperator);
1147 1148 1149 1150 1151 1152 1153
  } else {
    pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
    if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
      return NULL;
    }

    blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
1154
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1155

1156
    if (pBlock->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1157 1158 1159
      doSetOperatorCompleted(pOperator);
    }

1160 1161 1162
    size_t rows = pBlock->info.rows;
    pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1163
    return (rows == 0) ? NULL : pBlock;
1164 1165 1166 1167
  }
}

// todo merged with the build group result.
1168
static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
1169 1170 1171 1172 1173 1174 1175 1176
                                  int32_t* rowCellInfoOffset) {
  size_t num = taosArrayGetSize(pUpdateList);

  for (int32_t i = 0; i < num; ++i) {
    SResKeyPos* pPos = taosArrayGetP(pUpdateList, i);

    SFilePage*  bufPage = getBufPage(pBuf, pPos->pos.pageId);
    SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
1177

1178
    for (int32_t j = 0; j < numOfOutput; ++j) {
1179
      SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, j, rowCellInfoOffset);
1180 1181
      if (pRow->numOfRows < pEntry->numOfRes) {
        pRow->numOfRows = pEntry->numOfRes;
1182 1183 1184 1185 1186 1187
      }
    }

    releaseBufPage(pBuf, bufPage);
  }
}
5
54liuyao 已提交
1188
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
L
Liu Jicong 已提交
1189
  for (int i = 0; i < num; i++) {
5
54liuyao 已提交
1190 1191
    if (type == STREAM_INVERT) {
      fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
L
Liu Jicong 已提交
1192
    } else if (type == STREAM_NORMAL) {
5
54liuyao 已提交
1193 1194 1195 1196
      fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
    }
  }
}
5
54liuyao 已提交
1197

1198
void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SOptrBasicInfo* pBinfo, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
1199
  SResultRow*     pResult = getResultRowByPos(pResultBuf, p1);
5
54liuyao 已提交
1200 1201
  SqlFunctionCtx* pCtx = pBinfo->pCtx;
  for (int32_t i = 0; i < numOfOutput; ++i) {
1202
    pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pBinfo->rowCellInfoOffset);
5
54liuyao 已提交
1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
      continue;
    }
    pResInfo->initialized = false;
    if (pCtx[i].functionId != -1) {
      pCtx[i].fpSet.init(&pCtx[i], pResInfo);
    }
  }
}

X
Xiaoyu Wang 已提交
1214 1215
void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, int16_t bytes, uint64_t groupId,
                   int32_t numOfOutput) {
5
54liuyao 已提交
1216 1217
  SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId);
  SResultRowPosition* p1 =
X
Xiaoyu Wang 已提交
1218
      (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1219 1220 1221 1222
  if (!p1) {
    // window has been closed
    return;
  }
5
54liuyao 已提交
1223 1224 1225
  doClearWindowImpl(p1, pSup->pResultBuf, pBinfo, numOfOutput);
}

X
Xiaoyu Wang 已提交
1226 1227
static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, SInterval* pInterval, int32_t tsIndex,
                           int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) {
5
54liuyao 已提交
1228
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
1229 1230
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
  int32_t          step = 0;
5
54liuyao 已提交
1231 1232 1233
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
X
Xiaoyu Wang 已提交
1234 1235
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL);
    step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
1236
    doClearWindow(pSup, pBinfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput);
1237 1238 1239
    if (pUpWins) {
      taosArrayPush(pUpWins, &win);
    }
5
54liuyao 已提交
1240 1241
  }
}
1242

5
54liuyao 已提交
1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259
static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) {
  void*  pIte = NULL;
  size_t keyLen = 0;
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    void*    key = taosHashGetKey(pIte, &keyLen);
    uint64_t groupId = *(uint64_t*)key;
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
    TSKEY          ts = *(int64_t*)((char*)key + sizeof(uint64_t));
    SResultRowPosition* pPos = (SResultRowPosition*)pIte;
    int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
1260 1261 1262 1263
bool isCloseWindow(STimeWindow *pWin, STimeWindowAggSupp* pSup) {
  return pWin->ekey < pSup->maxTs - pSup->waterMark;
}

X
Xiaoyu Wang 已提交
1264 1265 1266
static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
                                   SArray* closeWins) {
  void*  pIte = NULL;
5
54liuyao 已提交
1267
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
1268 1269 1270
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    void*    key = taosHashGetKey(pIte, &keyLen);
    uint64_t groupId = *(uint64_t*)key;
5
54liuyao 已提交
1271
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
X
Xiaoyu Wang 已提交
1272
    TSKEY          ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1273 1274
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
X
Xiaoyu Wang 已提交
1275
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL);
5
54liuyao 已提交
1276
    if (isCloseWindow(&win, pSup)) {
5
54liuyao 已提交
1277 1278
      char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
      SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
1279
      taosHashRemove(pHashMap, keyBuf, keyLen);
5
54liuyao 已提交
1280 1281 1282 1283 1284 1285
      SResultRowPosition* pPos = (SResultRowPosition*)pIte;
      if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
        int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins);
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
5
54liuyao 已提交
1286 1287 1288 1289 1290 1291
      }
    }
  }
  return TSDB_CODE_SUCCESS;
}

1292
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
1293
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
1294
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1295 1296

  pInfo->order = TSDB_ORDER_ASC;
1297 1298 1299 1300 1301 1302

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
1303
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1304
    if (pInfo->binfo.pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1305 1306 1307 1308 1309 1310 1311
      pOperator->status = OP_EXEC_DONE;
    }
    return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];

5
54liuyao 已提交
1312
  SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);
1313
  while (1) {
1314
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1315 1316 1317 1318
    if (pBlock == NULL) {
      break;
    }

5
54liuyao 已提交
1319
    if (pBlock->info.type == STREAM_REPROCESS) {
X
Xiaoyu Wang 已提交
1320
      doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, 0, pOperator->numOfExprs, pBlock, NULL);
1321
      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
5
54liuyao 已提交
1322
      continue;
1323
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
1324 1325
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
      continue;
5
54liuyao 已提交
1326
    }
1327

1328 1329 1330 1331 1332 1333 1334 1335
    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
    // caller. Note that all the time window are not close till now.
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
    if (pInfo->invertible) {
      setInverFunction(pInfo->binfo.pCtx, pOperator->numOfExprs, pBlock->info.type);
    }

5
54liuyao 已提交
1336
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
H
Haojun Liao 已提交
1337
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated);
1338
  }
5
54liuyao 已提交
1339
  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
1340

1341
  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset);
1342 1343
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
1344
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357

  pOperator->status = OP_RES_TO_RETURN;

  return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}

static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  taosMemoryFreeClear(pInfo->stateKey.pData);
}

void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) {
1358
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
1359 1360 1361 1362
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  cleanupAggSup(&pInfo->aggSup);
}

5
54liuyao 已提交
1363
void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
1364
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
5
54liuyao 已提交
1365 1366
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  cleanupAggSup(&pInfo->aggSup);
1367 1368 1369 1370 1371 1372 1373 1374 1375
  if (pInfo->pChildren) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
      destroyIntervalOperatorInfo(pChildOp->info, numOfOutput);
      taosMemoryFreeClear(pChildOp->info);
      taosMemoryFreeClear(pChildOp);
    }
  }
1376
  nodesDestroyNode((SNode*)pInfo->pPhyNode);
5
54liuyao 已提交
1377 1378
}

1379
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
5
54liuyao 已提交
1380 1381 1382 1383 1384 1385 1386 1387
  for (int32_t i = 0; i < numOfCols; i++) {
    if (!fmIsInvertible(pFCtx[i].functionId)) {
      return false;
    }
  }
  return true;
}

1388
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
1389 1390
  // the primary timestamp column
  bool needed = false;
1391 1392
  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
1393

X
Xiaoyu Wang 已提交
1394
  {  // ts column
1395 1396
    SColumn c = {0};
    c.colId = 1;
1397
    c.slotId = pInfo->primaryTsIndex;
1398 1399
    c.type = TSDB_DATA_TYPE_TIMESTAMP;
    c.bytes = sizeof(int64_t);
1400
    taosArrayPush(pInfo->pInterpCols, &c);
1401 1402

    SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1403 1404 1405 1406
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = true;  // to denote no value is assigned yet
    key.pData = taosMemoryCalloc(1, c.bytes);
1407
    taosArrayPush(pInfo->pPrevValues, &key);
1408 1409
  }

X
Xiaoyu Wang 已提交
1410
  for (int32_t i = 0; i < numOfCols; ++i) {
1411 1412
    SExprInfo* pExpr = pCtx[i].pExpr;

H
Haojun Liao 已提交
1413
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1414 1415 1416
      SFunctParam* pParam = &pExpr->base.pParam[0];

      SColumn c = *pParam->pCol;
1417
      taosArrayPush(pInfo->pInterpCols, &c);
1418 1419 1420
      needed = true;

      SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1421 1422
      key.bytes = c.bytes;
      key.type = c.type;
1423
      key.isNull = false;
X
Xiaoyu Wang 已提交
1424
      key.pData = taosMemoryCalloc(1, c.bytes);
1425
      taosArrayPush(pInfo->pPrevValues, &key);
1426 1427 1428 1429 1430 1431
    }
  }

  return needed;
}

1432 1433 1434 1435 1436 1437
void increaseTs(SqlFunctionCtx* pCtx) {
  if (pCtx[0].pExpr->pExpr->_function.pFunctNode->funcType == FUNCTION_TYPE_WSTARTTS) {
    pCtx[0].increase = true;
  }
}

1438 1439
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                          SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
1440
                                          STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream) {
1441
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1442
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1443 1444 1445 1446
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

X
Xiaoyu Wang 已提交
1447 1448 1449
  pInfo->win = pTaskInfo->window;
  pInfo->order = TSDB_ORDER_ASC;
  pInfo->interval = *pInterval;
L
Liu Jicong 已提交
1450
  pInfo->execModel = pTaskInfo->execModel;
X
Xiaoyu Wang 已提交
1451
  pInfo->twAggSup = *pTwAggSupp;
1452

1453 1454
  pInfo->primaryTsIndex = primaryTsSlotId;

1455 1456
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);
1457 1458 1459

  int32_t code =
      initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
1460 1461 1462 1463 1464
  
  if (isStream) {
    ASSERT(numOfCols > 0);
    increaseTs(pInfo->binfo.pCtx);
  }
1465

1466
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
1467

5
54liuyao 已提交
1468
  pInfo->invertible = allInvertible(pInfo->binfo.pCtx, numOfCols);
X
Xiaoyu Wang 已提交
1469
  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
1470

1471
  pInfo->timeWindowInterpo = timeWindowinterpNeeded(pInfo->binfo.pCtx, numOfCols, pInfo);
1472 1473
  if (pInfo->timeWindowInterpo) {
    pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
H
Haojun Liao 已提交
1474 1475 1476
    if (pInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
1477 1478
  }

1479
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1480

X
Xiaoyu Wang 已提交
1481 1482 1483 1484 1485 1486 1487 1488
  pOperator->name = "TimeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->numOfExprs = numOfCols;
  pOperator->info = pInfo;
1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499

  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL,
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1500
_error:
1501 1502 1503 1504 1505 1506 1507 1508 1509
  destroyIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                                SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
wmmhello's avatar
wmmhello 已提交
1510
                                                STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
1511
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1512
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

  pInfo->order = TSDB_ORDER_ASC;
  pInfo->interval = *pInterval;
  pInfo->execModel = OPTR_EXEC_MODEL_STREAM;
  pInfo->win = pTaskInfo->window;
  pInfo->twAggSup = *pTwAggSupp;
  pInfo->primaryTsIndex = primaryTsSlotId;

  int32_t numOfRows = 4096;
  size_t  keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

  initResultSizeInfo(pOperator, numOfRows);
  int32_t code =
      initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);

wmmhello's avatar
wmmhello 已提交
1532
  if (code != TSDB_CODE_SUCCESS) {
1533 1534 1535
    goto _error;
  }

1536
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1537 1538

  pOperator->name = "StreamTimeIntervalAggOperator";
X
Xiaoyu Wang 已提交
1539
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
1540
  pOperator->blocking = true;
1541 1542 1543
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
1544
  pOperator->numOfExprs = numOfCols;
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556
  pOperator->info = pInfo;

  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL,
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1557
_error:
1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568
  destroyIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

// todo handle multiple tables cases.
static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

1569
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1570 1571

  bool    masterScan = true;
1572
  int32_t numOfOutput = pOperator->numOfExprs;
1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
  int64_t gid = pBlock->info.groupId;

  int64_t gap = pInfo->gap;

  if (!pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->winSup.prevTs = INT64_MIN;
  }

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

  // In case of ascending or descending order scan data, only one time window needs to be kepted for each table.
  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
    if (pInfo->winSup.prevTs == INT64_MIN) {
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
    } else if (tsList[j] - pRowSup->prevTs <= gap && (tsList[j] - pRowSup->prevTs) >= 0) {
      // The gap is less than the threshold, so it belongs to current session window that has been opened already.
      doKeepTuple(pRowSup, tsList[j]);
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
      int32_t ret =
          setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pInfo->binfo.pCtx,
                                 numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
      }

      // pInfo->numOfRows data belong to the current session window
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1613
      doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631
                       pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);

      // here we start a new session window
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
  int32_t ret =
      setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pInfo->binfo.pCtx,
                             numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
1632
  doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1633 1634 1635
                   pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
}

1636
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
1637 1638 1639 1640 1641 1642 1643 1644
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*          pBInfo = &pInfo->binfo;

  if (pOperator->status == OP_RES_TO_RETURN) {
1645
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1646
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1647 1648 1649 1650 1651 1652 1653
      doSetOperatorCompleted(pOperator);
      return NULL;
    }

    return pBInfo->pRes;
  }

1654 1655 1656
  int64_t st = taosGetTimestampUs();
  int32_t order = TSDB_ORDER_ASC;

1657 1658 1659
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1660
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1661 1662 1663 1664 1665
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1666
    setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, MAIN_SCAN, true);
1667 1668
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1669 1670 1671
    doSessionWindowAggImpl(pOperator, pInfo, pBlock);
  }

1672 1673
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;

1674 1675 1676 1677
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pBInfo->resultRowInfo);

1678
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1679
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1680
  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1681
  if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1682 1683 1684
    doSetOperatorCompleted(pOperator);
  }

1685 1686 1687
  size_t rows = pBInfo->pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1688
  return (rows == 0) ? NULL : pBInfo->pRes;
1689 1690
}

H
Haojun Liao 已提交
1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock) {
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for(int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData*  pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);

      pkey->isNull = false;
      char* val = colDataGetData(pColInfoData, i);
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }
}

H
Haojun Liao 已提交
1707
static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
1708 1709 1710 1711 1712
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
H
Haojun Liao 已提交
1713 1714
  SSDataBlock* pResBlock = pSliceInfo->binfo.pRes;

H
Haojun Liao 已提交
1715 1716
//  if (pOperator->status == OP_RES_TO_RETURN) {
//    //    doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes);
1717
//    if (pResBlock->info.rows == 0 || !hasDataInGroupInfo(&pSliceInfo->groupResInfo)) {
H
Haojun Liao 已提交
1718 1719 1720 1721 1722
//      doSetOperatorCompleted(pOperator);
//    }
//
//    return pResBlock;
//  }
1723

H
Haojun Liao 已提交
1724 1725
  int32_t order = TSDB_ORDER_ASC;
  SInterval* pInterval = &pSliceInfo->interval;
1726 1727
  SOperatorInfo* downstream = pOperator->pDownstream[0];

H
Haojun Liao 已提交
1728
  int32_t numOfRows = 0;
1729
  while (1) {
1730
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1731 1732 1733 1734 1735
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1736
    setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
H
Haojun Liao 已提交
1737 1738 1739 1740 1741 1742

    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, 0);
    for(int32_t i = 0; i < pBlock->info.rows; ++i) {
      int64_t ts = *(int64_t*) colDataGetData(pTsCol, i);

      if (ts == pSliceInfo->current) {
H
Haojun Liao 已提交
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755
        for(int32_t j = 0; j < pOperator->numOfExprs; ++j) {
          SExprInfo* pExprInfo = &pOperator->pExpr[j];
          int32_t dstSlot = pExprInfo->base.resSchema.slotId;
          int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;

          SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
          SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, dstSlot);

          char* v = colDataGetData(pSrc, i);
          colDataAppend(pDst, numOfRows, v, false);
        }

        numOfRows += 1;
H
Haojun Liao 已提交
1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766

        pSliceInfo->current += taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
        if (pSliceInfo->current > pSliceInfo->win.ekey) {
          doSetOperatorCompleted(pOperator);
          break;
        }
      } else if (ts < pSliceInfo->current) {
        if (i != pBlock->info.window.ekey) {
          int64_t nextTs = *(int64_t*) colDataGetData(pTsCol, i + 1);
          if (nextTs > pSliceInfo->current) {
            // output the result
H
Haojun Liao 已提交
1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850
            for (int32_t j = 0; j < pOperator->numOfExprs; ++j) {
              SExprInfo* pExprInfo = &pOperator->pExpr[j];
              int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
              int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;

              SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
              SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, dstSlot);

              switch (pSliceInfo->fillType) {
                case TSDB_FILL_NULL:
                  colDataAppendNULL(pDst, numOfRows);
                  break;

                case TSDB_FILL_SET_VALUE: {
                  SVariant* pVar = &pSliceInfo->pFillColInfo[i].fillVal;

                  if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
                    float v = 0;
                    GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
                    colDataAppend(pDst, numOfRows, (char*)&v, false);
                  } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
                    double v = 0;
                    GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
                    colDataAppend(pDst, numOfRows, (char*)&v, false);
                  } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
                    int64_t v = 0;
                    GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
                    colDataAppend(pDst, numOfRows, (char*)&v, false);
                  }
                }
                break;

                case TSDB_FILL_LINEAR:
#if 0
                if (pCtx->start.key == INT64_MIN || pCtx->start.key > pCtx->startTs
                    || pCtx->end.key == INT64_MIN || pCtx->end.key < pCtx->startTs) {
//                  goto interp_exit;
                }

              double v1 = -1, v2 = -1;
              GET_TYPED_DATA(v1, double, pCtx->inputType, &pCtx->start.val);
              GET_TYPED_DATA(v2, double, pCtx->inputType, &pCtx->end.val);

              SPoint point1 = {.key = ts, .val = &v1};
              SPoint point2 = {.key = nextTs, .val = &v2};
              SPoint point  = {.key = pCtx->startTs, .val = pCtx->pOutput};

              int32_t srcType = pCtx->inputType;
              if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) {
                setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
              } else {
                bool exceedMax = false, exceedMin = false;
                taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE, &exceedMax, &exceedMin);
                if (exceedMax || exceedMin) {
                  __compar_fn_t func = getComparFunc((int32_t)pCtx->inputType, 0);
                  if (func(&pCtx->start.val, &pCtx->end.val) <= 0) {
                    COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->start.val : &pCtx->end.val);
                  } else {
                    COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->end.val : &pCtx->start.val);
                  }
                }
              }
#endif
                  break;

                case TSDB_FILL_PREV: {
                  SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
                  colDataAppend(pDst, numOfRows, pkey->pData, false);
                } break;

                case TSDB_FILL_NEXT: {
                } break;

                case TSDB_FILL_NONE:
                default:
                  break;
              }

              pSliceInfo->current +=
                  taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
H
Haojun Liao 已提交
1851 1852
            }
          } else {
H
Haojun Liao 已提交
1853
            // ignore current row, and do nothing
H
Haojun Liao 已提交
1854 1855
          }
        } else {  // it is the last row of current block
H
Haojun Liao 已提交
1856
          doKeepPrevRows(pSliceInfo, pBlock);
H
Haojun Liao 已提交
1857 1858 1859
        }
      }
    }
1860 1861 1862 1863
  }

  // restore the value
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
1864
  if (pResBlock->info.rows == 0) {
1865 1866 1867
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895
  return pResBlock->info.rows == 0 ? NULL : pResBlock;
}

static int32_t initTimesliceInfo(STimeSliceOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfCols) {
  pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
  pInfo->pCols = taosArrayInit(4, sizeof(SColumn));

  if (pInfo->pPrevRow == NULL || pInfo->pCols == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  for (int32_t i = 0; i < numOfCols; ++i) {
    SExprInfo* pExpr = pCtx[i].pExpr;

    SFunctParam* pParam = &pExpr->base.pParam[0];

    SColumn c = *pParam->pCol;
    taosArrayPush(pInfo->pCols, &c);

    SGroupKeys key = {0};
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, c.bytes);
    taosArrayPush(pInfo->pPrevRow, &key);
  }

  return TSDB_CODE_SUCCESS;
1896 1897 1898
}

SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
H
Haojun Liao 已提交
1899
                                           SSDataBlock* pResultBlock, const SNodeListNode* pValNode, SExecTaskInfo* pTaskInfo) {
1900 1901 1902 1903 1904 1905
  STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pOperator == NULL || pInfo == NULL) {
    goto _error;
  }

H
Haojun Liao 已提交
1906 1907 1908 1909 1910
  int32_t code = initTimesliceInfo(pInfo, pInfo->binfo.pCtx, numOfCols);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

1911
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
H
Haojun Liao 已提交
1912
  pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfCols, pValNode);
1913

H
Haojun Liao 已提交
1914
  pInfo->binfo.pRes     = pResultBlock;
H
Haojun Liao 已提交
1915 1916

  pOperator->name       = "TimeSliceOperator";
1917
  //  pOperator->operatorType = OP_AllTimeWindow;
H
Haojun Liao 已提交
1918 1919 1920
  pOperator->blocking   = true;
  pOperator->status     = OP_NOT_OPENED;
  pOperator->pExpr      = pExprInfo;
1921
  pOperator->numOfExprs = numOfCols;
H
Haojun Liao 已提交
1922 1923
  pOperator->info       = pInfo;
  pOperator->pTaskInfo  = pTaskInfo;
1924

H
Haojun Liao 已提交
1925
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyBasicOperatorInfo,
1926 1927
                                         NULL, NULL, NULL);

H
Haojun Liao 已提交
1928
  code = appendDownstream(pOperator, &downstream, 1);
1929 1930
  return pOperator;

L
Liu Jicong 已提交
1931
_error:
1932 1933 1934 1935 1936 1937 1938
  taosMemoryFree(pInfo);
  taosMemoryFree(pOperator);
  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
  return NULL;
}

SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
1939
                                             SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId,
1940
                                             SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo) {
1941 1942 1943 1944 1945 1946
  SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1947 1948 1949 1950 1951 1952 1953 1954
  pInfo->stateCol = *pStateKeyCol;
  pInfo->stateKey.type = pInfo->stateCol.type;
  pInfo->stateKey.bytes = pInfo->stateCol.bytes;
  pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
  if (pInfo->stateKey.pData == NULL) {
    goto _error;
  }

1955 1956 1957 1958
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

  initResultSizeInfo(pOperator, 4096);
  initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
1959
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1960

L
Liu Jicong 已提交
1961
  pInfo->twAggSup = *pTwAggSup;
1962 1963
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

X
Xiaoyu Wang 已提交
1964 1965
  pInfo->tsSlotId = tsSlotId;
  pOperator->name = "StateWindowOperator";
1966
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE;
X
Xiaoyu Wang 已提交
1967 1968 1969
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExpr;
L
Liu Jicong 已提交
1970
  pOperator->numOfExprs = numOfCols;
X
Xiaoyu Wang 已提交
1971 1972
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
1973 1974 1975 1976 1977 1978 1979

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL,
                                         destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  int32_t code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

L
Liu Jicong 已提交
1980
_error:
1981 1982 1983 1984 1985 1986 1987 1988 1989 1990
  pTaskInfo->code = TSDB_CODE_SUCCESS;
  return NULL;
}

void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
}

SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
L
Liu Jicong 已提交
1991 1992
                                            SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId,
                                            STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
1993 1994 1995 1996 1997 1998
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1999 2000
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);
2001 2002 2003 2004 2005 2006 2007 2008

  int32_t code =
      initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->twAggSup = *pTwAggSupp;
2009
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2010 2011
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

L
Liu Jicong 已提交
2012 2013 2014 2015 2016 2017
  pInfo->tsSlotId = tsSlotId;
  pInfo->gap = gap;
  pInfo->binfo.pRes = pResBlock;
  pInfo->winSup.prevTs = INT64_MIN;
  pInfo->reptScan = false;
  pOperator->name = "SessionWindowAggOperator";
2018
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
2019
  pOperator->blocking = true;
L
Liu Jicong 已提交
2020 2021 2022 2023
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->numOfExprs = numOfCols;
  pOperator->info = pInfo;
2024 2025 2026 2027 2028 2029 2030 2031

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL,
                                         destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  pOperator->pTaskInfo = pTaskInfo;

  code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

L
Liu Jicong 已提交
2032
_error:
2033 2034 2035 2036 2037 2038 2039 2040
  if (pInfo != NULL) {
    destroySWindowOperatorInfo(pInfo, numOfCols);
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
2041
}
5
54liuyao 已提交
2042

5
54liuyao 已提交
2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091
bool isFinalInterval(SStreamFinalIntervalOperatorInfo* pInfo) { return pInfo->pChildren != NULL; }

void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
                      SExecTaskInfo* pTaskInfo) {
  for (int32_t k = 0; k < numOfOutput; ++k) {
    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
      continue;
    }
    int32_t code = TSDB_CODE_SUCCESS;
    if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
      code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
      if (code != TSDB_CODE_SUCCESS) {
        qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
        pTaskInfo->code = code;
        longjmp(pTaskInfo->env, code);
      }
    }
  }
}

static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId,
                                  int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
  int32_t size = taosArrayGetSize(pWinArray);
  ASSERT(pInfo->pChildren);
  for (int32_t i = 0; i < size; i++) {
    STimeWindow* pParentWin = taosArrayGet(pWinArray, i);
    SResultRow*  pCurResult = NULL;
    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, pParentWin, true, &pCurResult, 0, pInfo->binfo.pCtx,
                           numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
    for (int32_t j = 0; j < numOfChildren; j++) {
      SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
      SResultRow*               pChResult = NULL;
      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, pParentWin, true, &pChResult, 0, pChInfo->binfo.pCtx,
                             pChildOp->numOfExprs, pChInfo->binfo.rowCellInfoOffset, &pChInfo->aggSup, pTaskInfo);
      compactFunctions(pInfo->binfo.pCtx, pChInfo->binfo.pCtx, numOfOutput, pTaskInfo);
    }
  }
}

bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
  SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId);
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable,
      pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(int64_t)));
  return p1 == NULL;
}

static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
S
shenglian zhou 已提交
2092
                           SArray* pUpdated) {
5
54liuyao 已提交
2093
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
X
Xiaoyu Wang 已提交
2094 2095 2096 2097 2098 2099 2100 2101
  SResultRowInfo*                   pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*                    pTaskInfo = pOperatorInfo->pTaskInfo;
  int32_t                           numOfOutput = pOperatorInfo->numOfExprs;
  int32_t                           step = 1;
  bool                              ascScan = true;
  TSKEY*                            tsCols = NULL;
  SResultRow*                       pResult = NULL;
  int32_t                           forwardRows = 0;
5
54liuyao 已提交
2102 2103 2104 2105

  if (pSDataBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
2106
  } else {
S
shenglian zhou 已提交
2107
    return;
5
54liuyao 已提交
2108
  }
5
54liuyao 已提交
2109

X
Xiaoyu Wang 已提交
2110 2111 2112 2113
  int32_t     startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
  STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
                                            pInfo->interval.precision, NULL);
5
54liuyao 已提交
2114
  while (1) {
5
54liuyao 已提交
2115 2116 2117 2118 2119 2120 2121 2122
    if (isFinalInterval(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) &&
        isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup)) {
      SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
      taosArrayPush(pUpWins, &nextWin);
      rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId,
          pOperatorInfo->numOfExprs, pOperatorInfo->pTaskInfo);
      taosArrayDestroy(pUpWins);
    }
X
Xiaoyu Wang 已提交
2123 2124
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx,
                                          numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2125 2126 2127 2128 2129 2130 2131
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    pos->groupId = tableGroupId;
    pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
    *(int64_t*)pos->key = pResult->win.skey;
S
shenglian zhou 已提交
2132 2133
    forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
                                           TSDB_ORDER_ASC);
5
54liuyao 已提交
2134
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) {
5
54liuyao 已提交
2135
      saveResultRow(pResult, tableGroupId, pUpdated);
5
54liuyao 已提交
2136
    }
5
54liuyao 已提交
2137
    // window start(end) key interpolation
S
shenglian zhou 已提交
2138 2139
    // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos,
    // forwardRows);
5
54liuyao 已提交
2140
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
X
Xiaoyu Wang 已提交
2141 2142
    doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
2143
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
2144
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
5
54liuyao 已提交
2145 2146 2147 2148 2149 2150 2151
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
    if (startPos < 0) {
      break;
    }
  }
}

5
54liuyao 已提交
2152 2153 2154 2155
static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) {
  taosHashClear(pInfo->aggSup.pResultRowHashTable);
  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
  cleanupResultRowInfo(&pInfo->binfo.resultRowInfo);
2156
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
2157 2158 2159 2160 2161 2162 2163 2164 2165
}

static void clearUpdateDataBlock(SSDataBlock* pBlock) {
  if (pBlock->info.rows <= 0) {
    return;
  }
  blockDataCleanup(pBlock);
}

2166
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
5
54liuyao 已提交
2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177
  ASSERT(pDest->info.capacity >= pSource->info.rows);
  clearUpdateDataBlock(pDest);
  SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
  SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
  // copy timestamp column
  colDataAssign(pDestCol, pSourceCol, pSource->info.rows);
  for (int32_t i = 1; i < pDest->info.numOfCols; i++) {
    SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
    colDataAppendNNULL(pCol, 0, pSource->info.rows);
  }
  pDest->info.rows = pSource->info.rows;
2178 2179
  pDest->info.groupId = pSource->info.groupId;
  pDest->info.type = pSource->info.type;
5
54liuyao 已提交
2180 2181 2182 2183
  blockDataUpdateTsWindow(pDest, 0);
}

static int32_t getChildIndex(SSDataBlock* pBlock) {
2184
  return pBlock->info.childId;
5
54liuyao 已提交
2185 2186
}

5
54liuyao 已提交
2187 2188
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
  SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2189
  SOperatorInfo*                    downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
2190
  SArray*                           pUpdated = taosArrayInit(4, POINTER_BYTES);
5
54liuyao 已提交
2191
  TSKEY                             maxTs = INT64_MIN;
5
54liuyao 已提交
2192 2193 2194 2195 2196

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
2197
    if (pInfo->binfo.pRes->info.rows == 0) {
5
54liuyao 已提交
2198
      pOperator->status = OP_EXEC_DONE;
5
54liuyao 已提交
2199 2200 2201 2202 2203 2204 2205 2206 2207 2208
      if (isFinalInterval(pInfo) || pInfo->pUpdateRes->info.rows == 0) {
        if (!isFinalInterval(pInfo)) {
          // semi interval operator clear disk buffer
          clearStreamIntervalOperator(pInfo);
        }
        return NULL;
      }
      // process the rest of the data
      pOperator->status = OP_OPENED;
      return pInfo->pUpdateRes;
5
54liuyao 已提交
2209
    }
5
54liuyao 已提交
2210
    return pInfo->binfo.pRes;
5
54liuyao 已提交
2211 2212 2213 2214 2215
  }

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
2216
      clearUpdateDataBlock(pInfo->pUpdateRes);
5
54liuyao 已提交
2217 2218
      break;
    }
2219

5
54liuyao 已提交
2220
    if (pBlock->info.type == STREAM_REPROCESS) {
X
Xiaoyu Wang 已提交
2221 2222 2223
      SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
      doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, pInfo->primaryTsIndex, pOperator->numOfExprs,
                     pBlock, pUpWins);
2224
      if (isFinalInterval(pInfo)) {
5
54liuyao 已提交
2225
        int32_t                   childIndex = getChildIndex(pBlock);
X
Xiaoyu Wang 已提交
2226
        SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
2227
        SIntervalAggOperatorInfo* pChildInfo = pChildOp->info;
S
shenglian zhou 已提交
2228 2229 2230 2231
        doClearWindows(&pChildInfo->aggSup, &pChildInfo->binfo, &pChildInfo->interval, pChildInfo->primaryTsIndex,
                       pChildOp->numOfExprs, pBlock, NULL);
        rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->numOfExprs,
                              pOperator->pTaskInfo);
5
54liuyao 已提交
2232 2233
        taosArrayDestroy(pUpWins);
        continue;
2234
      }
5
54liuyao 已提交
2235 2236
      removeResults(pUpWins, pUpdated);
      copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
2237
      taosArrayDestroy(pUpWins);
5
54liuyao 已提交
2238
      break;
2239
    } else if (pBlock->info.type == STREAM_GET_ALL && isFinalInterval(pInfo)) {
5
54liuyao 已提交
2240 2241
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
      continue;
5
54liuyao 已提交
2242
    }
5
54liuyao 已提交
2243

2244
    setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
5
54liuyao 已提交
2245
    doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated);
2246
    if (isFinalInterval(pInfo)) {
S
shenglian zhou 已提交
2247
      int32_t chIndex = getChildIndex(pBlock);
5
54liuyao 已提交
2248 2249 2250 2251 2252 2253 2254 2255 2256
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
        SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
        if (!pChildOp) {
          longjmp(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
        }
        taosArrayPush(pInfo->pChildren, &pChildOp);
      }
S
shenglian zhou 已提交
2257
      SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
2258 2259 2260 2261
      SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
      setInputDataBlock(pChildOp, pChInfo->binfo.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
      doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL);
    }
5
54liuyao 已提交
2262
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
2263
  }
S
shenglian zhou 已提交
2264

5
54liuyao 已提交
2265
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
2266
  if (isFinalInterval(pInfo)) {
5
54liuyao 已提交
2267
    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
5
54liuyao 已提交
2268 2269 2270 2271 2272 2273 2274
  }

  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset);
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
  pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286
  if (pInfo->binfo.pRes->info.rows == 0) {
    pOperator->status = OP_EXEC_DONE;
    if (pInfo->pUpdateRes->info.rows == 0) {
      return NULL;
    }
    // process the rest of the data
    pOperator->status = OP_OPENED;
    return pInfo->pUpdateRes;
  }
  return pInfo->binfo.pRes;
}

S
shenglian zhou 已提交
2287 2288 2289
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                     SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
  SIntervalPhysiNode*               pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
5
54liuyao 已提交
2290
  SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo));
S
shenglian zhou 已提交
2291
  SOperatorInfo*                    pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
2292 2293 2294 2295
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
  pInfo->order = TSDB_ORDER_ASC;
S
shenglian zhou 已提交
2296 2297 2298 2299 2300 2301 2302 2303
  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                .sliding = pIntervalPhyNode->sliding,
                                .intervalUnit = pIntervalPhyNode->intervalUnit,
                                .slidingUnit = pIntervalPhyNode->slidingUnit,
                                .offset = pIntervalPhyNode->offset,
                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pIntervalPhyNode->window.watermark,
5
54liuyao 已提交
2304 2305
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
S
shenglian zhou 已提交
2306
  };
2307
  ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY);
5
54liuyao 已提交
2308 2309 2310
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);
S
shenglian zhou 已提交
2311 2312
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5
54liuyao 已提交
2313 2314 2315
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols,
      pResBlock, keyBufSize, pTaskInfo->id.str);
2316 2317
  ASSERT(numOfCols > 0);
  increaseTs(pInfo->binfo.pCtx);
5
54liuyao 已提交
2318 2319 2320 2321
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
2322
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338
  pInfo->pChildren = NULL;
  if (numOfChild > 0) {
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(SOperatorInfo));
    for (int32_t i = 0; i < numOfChild; i++) {
      SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp) {
        taosArrayPush(pInfo->pChildren, &pChildOp);
        continue;
      }
      goto _error;
    }
  }
  // semi interval operator does not catch result
  if (!isFinalInterval(pInfo)) {
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }
S
shenglian zhou 已提交
2339
  pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
5
54liuyao 已提交
2340 2341
  pInfo->pUpdateRes->info.type = STREAM_REPROCESS;
  blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
2342
  pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
5
54liuyao 已提交
2343 2344 2345 2346 2347 2348 2349 2350 2351 2352

  pOperator->name = "StreamFinalIntervalOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->numOfExprs = numOfCols;
  pOperator->info = pInfo;

S
shenglian zhou 已提交
2353 2354 2355
  pOperator->fpSet =
      createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
  destroyStreamFinalIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
5
54liuyao 已提交
2370
}
5
54liuyao 已提交
2371 2372 2373

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
2374 2375 2376 2377 2378 2379
  void **pIte = NULL;
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
    SArray *pWins = (SArray *) (*pIte);
    taosArrayDestroy(pWins);
  }
  taosHashCleanup(pSup->pResultRows);
5
54liuyao 已提交
2380 2381 2382 2383 2384 2385 2386 2387
  destroyDiskbasedBuf(pSup->pResultBuf);
}

void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
2388 2389 2390
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
2391
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
2392 2393 2394 2395 2396 2397
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
      destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput);
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
5
54liuyao 已提交
2398 2399
}

S
shenglian zhou 已提交
2400
int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock) {
5
54liuyao 已提交
2401 2402 2403
  pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset);
  pBasicInfo->pRes = pResultBlock;
  for (int32_t i = 0; i < numOfCols; ++i) {
2404
    pBasicInfo->pCtx[i].pBuf = NULL;
5
54liuyao 已提交
2405
  }
2406 2407
  ASSERT(numOfCols > 0);
  increaseTs(pBasicInfo->pCtx);
5
54liuyao 已提交
2408 2409 2410 2411 2412 2413 2414 2415
  return TSDB_CODE_SUCCESS;
}

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {
  for (int i = 0; i < nums; i++) {
    pDummy[i].functionId = pCtx[i].functionId;
  }
}
X
Xiaoyu Wang 已提交
2416 2417
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
                    uint8_t type) {
5
54liuyao 已提交
2418 2419
  ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
  SStreamBlockScanInfo* pScanInfo = downstream->info;
X
Xiaoyu Wang 已提交
2420
  pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
5
54liuyao 已提交
2421
  pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
5
54liuyao 已提交
2422 2423
}

2424 2425 2426 2427
int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo));
}

2428 2429 2430 2431 2432 2433 2434
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
  SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
  int32_t numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  int32_t code = TSDB_CODE_OUT_OF_MEMORY;
X
Xiaoyu Wang 已提交
2435
  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
2436
  SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
2437 2438 2439 2440 2441 2442
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

  initResultSizeInfo(pOperator, 4096);

2443
  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
2444 2445 2446
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
2447 2448
  
  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pInfo->binfo.pCtx, numOfCols);
5
54liuyao 已提交
2449 2450 2451
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
X
Xiaoyu Wang 已提交
2452

5
54liuyao 已提交
2453 2454 2455 2456 2457 2458
  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }
  initDummyFunction(pInfo->pDummyCtx, pInfo->binfo.pCtx, numOfCols);

H
Haojun Liao 已提交
2459 2460
  pInfo->twAggSup = (STimeWindowAggSupp) {
      .waterMark = pSessionNode->window.watermark,
2461 2462
      .calTrigger = pSessionNode->window.triggerType,
      .maxTs = INT64_MIN};
H
Haojun Liao 已提交
2463 2464

  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
2465 2466 2467
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

  pInfo->primaryTsIndex = tsSlotId;
2468
  pInfo->gap = pSessionNode->gap;
5
54liuyao 已提交
2469 2470 2471 2472 2473 2474 2475
  pInfo->binfo.pRes = pResBlock;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
  pInfo->pDelRes = createOneDataBlock(pResBlock, false);
  blockDataEnsureCapacity(pInfo->pDelRes, 64);
2476
  pInfo->pChildren = NULL;
5
54liuyao 已提交
2477 2478

  pOperator->name = "StreamSessionWindowAggOperator";
2479
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
5
54liuyao 已提交
2480 2481 2482 2483 2484
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->numOfExprs = numOfCols;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
2485 2486 2487
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
2488
  pOperator->pTaskInfo = pTaskInfo;
X
Xiaoyu Wang 已提交
2489
  initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType);
5
54liuyao 已提交
2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504
  code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

_error:
  if (pInfo != NULL) {
    destroyStreamSessionAggOperatorInfo(pInfo, numOfCols);
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

int64_t getSessionWindowEndkey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
2505
  SArray*            pWinInfos = (SArray*)data;
5
54liuyao 已提交
2506 2507 2508 2509 2510 2511
  SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index);
  return pWin->win.ekey;
}
static bool isInWindow(SResultWindowInfo* pWin, TSKEY ts, int64_t gap) {
  int64_t sGap = ts - pWin->win.skey;
  int64_t eGap = pWin->win.ekey - ts;
X
Xiaoyu Wang 已提交
2512
  if ((sGap < 0 && sGap >= -gap) || (eGap < 0 && eGap >= -gap) || (sGap >= 0 && eGap >= 0)) {
5
54liuyao 已提交
2513 2514 2515 2516 2517
    return true;
  }
  return false;
}

X
Xiaoyu Wang 已提交
2518 2519
static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) {
  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
5
54liuyao 已提交
2520 2521 2522 2523
  return taosArrayInsert(pWinInfos, index, &win);
}

static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) {
X
Xiaoyu Wang 已提交
2524
  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
5
54liuyao 已提交
2525 2526 2527
  return taosArrayPush(pWinInfos, &win);
}

2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543
SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) {
  void** ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t));
  SArray* pWinInfos = NULL;
  if (ite == NULL) {
    pWinInfos = taosArrayInit(1024, pAggSup->valueSize);
    taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void *));
  } else {
    pWinInfos = *ite;
  }
  return pWinInfos;
}

SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int64_t gap, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

5
54liuyao 已提交
2544 2545
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
5
54liuyao 已提交
2546
    *pIndex = 0;
5
54liuyao 已提交
2547 2548 2549
    return addNewSessionWindow(pWinInfos, ts);
  }
  // find the first position which is smaller than the key
X
Xiaoyu Wang 已提交
2550
  int32_t            index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isInWindow(pWin, ts, gap)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isInWindow(pWin, ts, gap)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewSessionWindow(pWinInfos, ts);
  }
5
54liuyao 已提交
2572 2573
  *pIndex = index + 1;
  return insertNewSessionWindow(pWinInfos, ts, index + 1);
5
54liuyao 已提交
2574 2575
}

X
Xiaoyu Wang 已提交
2576 2577
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, int32_t start, int64_t gap,
                                SHashObj* pStDeleted) {
5
54liuyao 已提交
2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593
  for (int32_t i = start; i < rows; ++i) {
    if (!isInWindow(pWinInfo, pTs[i], gap)) {
      return i - start;
    }
    if (pWinInfo->win.skey > pTs[i]) {
      if (pStDeleted && pWinInfo->isOutput) {
        taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY));
        pWinInfo->isOutput = false;
      }
      pWinInfo->win.skey = pTs[i];
    }
    pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pTs[i]);
  }
  return rows - start;
}

X
Xiaoyu Wang 已提交
2594
static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,
5
54liuyao 已提交
2595
                                  uint64_t groupId, int32_t numOfOutput, int32_t* rowCellInfoOffset,
X
Xiaoyu Wang 已提交
2596
                                  SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) {
5
54liuyao 已提交
2597 2598
  assert(pWinInfo->win.skey <= pWinInfo->win.ekey);
  // too many time window in query
2599
  int32_t size = taosArrayGetSize(pAggSup->pCurWins);
5
54liuyao 已提交
2600 2601 2602
  if (size > MAX_INTERVAL_TIME_WINDOW) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
  }
X
Xiaoyu Wang 已提交
2603

5
54liuyao 已提交
2604
  if (pWinInfo->pos.pageId == -1) {
2605
    *pResult = getNewResultRow(pAggSup->pResultBuf, groupId, pAggSup->resultRowSize);
5
54liuyao 已提交
2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627
    if (*pResult == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }
    initResultRow(*pResult);

    // add a new result set for a new group
    pWinInfo->pos.pageId = (*pResult)->pageId;
    pWinInfo->pos.offset = (*pResult)->offset;
  } else {
    *pResult = getResultRowByPos(pAggSup->pResultBuf, &pWinInfo->pos);
    if (!(*pResult)) {
      qError("getResultRowByPos return NULL, TID:%s", GET_TASKID(pTaskInfo));
      return TSDB_CODE_FAILED;
    }
  }

  // set time window for current result
  (*pResult)->win = pWinInfo->win;
  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowCellInfoOffset);
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
2628 2629 2630 2631 2632 2633 2634 2635
static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup,
                                  SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock,
                                  SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
                                  int32_t numOutput, SExecTaskInfo* pTaskInfo) {
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId);
  TSKEY*           tsCols = (int64_t*)pColDataInfo->pData;
  int32_t          code = setWindowOutputBuf(pCurWin, pResult, pBinfo->pCtx, pSDataBlock->info.groupId, numOutput,
                                             pBinfo->rowCellInfoOffset, pAggSup, pTaskInfo);
5
54liuyao 已提交
2636 2637 2638
  if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }
5
54liuyao 已提交
2639
  updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, true);
X
Xiaoyu Wang 已提交
2640 2641
  doApplyFunctions(pTaskInfo, pBinfo->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols,
                   pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC);
5
54liuyao 已提交
2642 2643 2644
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
2645 2646 2647 2648 2649
static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                              SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
                              int32_t numOutput, SExecTaskInfo* pTaskInfo) {
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pTaskInfo);
5
54liuyao 已提交
2650 2651
}

X
Xiaoyu Wang 已提交
2652 2653 2654 2655 2656
static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                                   SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex,
                                   int32_t winRows, int32_t numOutput, SExecTaskInfo* pTaskInfo) {
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pTaskInfo);
5
54liuyao 已提交
2657 2658
}

5
54liuyao 已提交
2659 2660
int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) {
  SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex);
X
Xiaoyu Wang 已提交
2661
  int32_t            size = taosArrayGetSize(pWinInfos);
5
54liuyao 已提交
2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i);
    if (!isInWindow(pCurWin, pWinInfo->win.skey, gap)) {
      return i - startIndex - 1;
    }
  }

  return size - startIndex - 1;
}

5
54liuyao 已提交
2673
void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId,
X
Xiaoyu Wang 已提交
2674
                       int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SHashObj* pStUpdated, SHashObj* pStDeleted) {
2675
  SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex);
X
Xiaoyu Wang 已提交
2676 2677 2678
  SResultRow*        pCurResult = NULL;
  setWindowOutputBuf(pCurWin, &pCurResult, pInfo->binfo.pCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset,
                     &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
2679
  num += startIndex + 1;
2680
  ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins));
5
54liuyao 已提交
2681 2682
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < num; i++) {
2683
    SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i);
X
Xiaoyu Wang 已提交
2684 2685 2686
    SResultRow*        pWinResult = NULL;
    setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset,
                       &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
2687 2688 2689 2690 2691 2692 2693
    pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey);
    compactFunctions(pInfo->binfo.pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo);
    taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition));
    if (pWinInfo->isOutput) {
      taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY));
      pWinInfo->isOutput = false;
    }
2694
    taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
5
54liuyao 已提交
2695 2696 2697
  }
}

5
54liuyao 已提交
2698 2699 2700 2701 2702
typedef struct SWinRes {
  TSKEY    ts;
  uint64_t groupId;
} SWinRes;

X
Xiaoyu Wang 已提交
2703 2704 2705
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
                                   SHashObj* pStDeleted) {
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
2706
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2707 2708
  bool                           masterScan = true;
  int32_t                        numOfOutput = pOperator->numOfExprs;
5
54liuyao 已提交
2709
  uint64_t                       groupId = pSDataBlock->info.groupId;
X
Xiaoyu Wang 已提交
2710 2711 2712 2713 2714 2715
  int64_t                        gap = pInfo->gap;
  int64_t                        code = TSDB_CODE_SUCCESS;

  int32_t     step = 1;
  bool        ascScan = true;
  TSKEY*      tsCols = NULL;
5
54liuyao 已提交
2716
  SResultRow* pResult = NULL;
X
Xiaoyu Wang 已提交
2717
  int32_t     winRows = 0;
5
54liuyao 已提交
2718 2719

  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
2720
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
2721 2722
    tsCols = (int64_t*)pColDataInfo->pData;
  } else {
X
Xiaoyu Wang 已提交
2723
    return;
5
54liuyao 已提交
2724
  }
X
Xiaoyu Wang 已提交
2725

5
54liuyao 已提交
2726
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
2727 2728
  for (int32_t i = 0; i < pSDataBlock->info.rows;) {
    int32_t            winIndex = 0;
5
54liuyao 已提交
2729
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], groupId, gap, &winIndex);
X
Xiaoyu Wang 已提交
2730
    winRows = updateSessionWindowInfo(pCurWin, tsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted);
5
54liuyao 已提交
2731 2732 2733 2734 2735
    code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    // window start(end) key interpolation
X
Xiaoyu Wang 已提交
2736 2737
    // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos,
    // forwardRows,
5
54liuyao 已提交
2738
    //                             pInfo->order, false);
2739
    int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap);
5
54liuyao 已提交
2740 2741 2742
    if (winNum > 0) {
      compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pTaskInfo, pStUpdated, pStDeleted);
    }
5
54liuyao 已提交
2743 2744
    pCurWin->isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
2745 2746
      SWinRes value = {.ts = pCurWin->win.skey, .groupId = groupId};
      code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinRes));
5
54liuyao 已提交
2747 2748 2749 2750
      if (code != TSDB_CODE_SUCCESS) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
      pCurWin->isOutput = true;
5
54liuyao 已提交
2751 2752 2753 2754 2755
    }
    i += winRows;
  }
}

X
Xiaoyu Wang 已提交
2756 2757
static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SOptrBasicInfo* pBinfo, SSDataBlock* pBlock,
                                  int32_t tsIndex, int32_t numOfOutput, int64_t gap, SArray* result) {
5
54liuyao 已提交
2758
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
2759 2760
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
  int32_t          step = 0;
5
54liuyao 已提交
2761
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
2762
    int32_t            winIndex = 0;
2763
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], pBlock->info.groupId, gap, &winIndex);
5
54liuyao 已提交
2764
    step = updateSessionWindowInfo(pCurWin, tsCols, pBlock->info.rows, i, gap, NULL);
2765
    ASSERT(isInWindow(pCurWin, tsCols[i], gap));
5
54liuyao 已提交
2766
    doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pBinfo, numOfOutput);
2767 2768 2769
    if (result) {
      taosArrayPush(result, pCurWin);
    }
5
54liuyao 已提交
2770 2771 2772
  }
}

5
54liuyao 已提交
2773
static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) {
X
Xiaoyu Wang 已提交
2774
  void*  pData = NULL;
5
54liuyao 已提交
2775
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
2776
  while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) {
5
54liuyao 已提交
2777 2778 2779 2780 2781 2782
    void* key = taosHashGetKey(pData, &keyLen);
    ASSERT(keyLen == sizeof(SResultRowPosition));
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    if (pos == NULL) {
      return TSDB_CODE_QRY_OUT_OF_MEMORY;
    }
5
54liuyao 已提交
2783
    pos->groupId = ((SWinRes*)pData)->groupId;
5
54liuyao 已提交
2784
    pos->pos = *(SResultRowPosition*)key;
5
54liuyao 已提交
2785
    *(int64_t*)pos->key = ((SWinRes*)pData)->ts;
5
54liuyao 已提交
2786 2787 2788 2789 2790 2791 2792 2793
    taosArrayPush(pUpdated, &pos);
  }
  return TSDB_CODE_SUCCESS;
}

void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
  blockDataCleanup(pBlock);
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
2794
  while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
5
54liuyao 已提交
2795
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
X
Xiaoyu Wang 已提交
2796
    colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false);
5
54liuyao 已提交
2797 2798
    for (int32_t i = 1; i < pBlock->info.numOfCols; i++) {
      pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
X
Xiaoyu Wang 已提交
2799
      colDataAppendNULL(pColInfoData, pBlock->info.rows);
5
54liuyao 已提交
2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810
    }
    pBlock->info.rows += 1;
    if (pBlock->info.rows + 1 >= pBlock->info.capacity) {
      break;
    }
  }
  if ((*Ite) == NULL) {
    taosHashClear(pStDeleted);
  }
}

X
Xiaoyu Wang 已提交
2811 2812
static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId,
                              int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
2813 2814 2815 2816
  int32_t size = taosArrayGetSize(pWinArray);
  ASSERT(pInfo->pChildren);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i);
X
Xiaoyu Wang 已提交
2817 2818 2819
    SResultRow*        pCurResult = NULL;
    setWindowOutputBuf(pParentWin, &pCurResult, pInfo->binfo.pCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset,
                       &pInfo->streamAggSup, pTaskInfo);
2820 2821
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
    for (int32_t j = 0; j < numOfChildren; j++) {
X
Xiaoyu Wang 已提交
2822
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
2823
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
2824
      SArray*                        pChWins = getWinInfos(&pChInfo->streamAggSup, groupId);
X
Xiaoyu Wang 已提交
2825 2826
      int32_t                        chWinSize = taosArrayGetSize(pChWins);
      int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey);
2827 2828 2829 2830
      for (int32_t k = index; k > 0 && k < chWinSize; k++) {
        SResultWindowInfo* pcw = taosArrayGet(pChWins, k);
        if (pParentWin->win.skey <= pcw->win.skey && pcw->win.ekey <= pParentWin->win.ekey) {
          SResultRow* pChResult = NULL;
X
Xiaoyu Wang 已提交
2831 2832
          setWindowOutputBuf(pcw, &pChResult, pChInfo->binfo.pCtx, groupId, numOfOutput,
                             pChInfo->binfo.rowCellInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
2833 2834 2835 2836 2837 2838 2839 2840 2841
          compactFunctions(pInfo->binfo.pCtx, pChInfo->binfo.pCtx, numOfOutput, pTaskInfo);
          continue;
        }
        break;
      }
    }
  }
}

X
Xiaoyu Wang 已提交
2842
bool isFinalSession(SStreamSessionAggOperatorInfo* pInfo) { return pInfo->pChildren != NULL; }
2843

X
Xiaoyu Wang 已提交
2844 2845 2846
typedef SResultWindowInfo* (*__get_win_info_)(void*);
SResultWindowInfo* getSessionWinInfo(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getStateWinInfo(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
5
54liuyao 已提交
2847

2848
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed,
5
54liuyao 已提交
2849
    __get_win_info_ fn) {
5
54liuyao 已提交
2850
  // Todo(liuyao) save window to tdb
2851
  void **pIte = NULL;
5
54liuyao 已提交
2852
  size_t keyLen = 0;
2853
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
5
54liuyao 已提交
2854
    uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen);
2855 2856 2857 2858 2859 2860 2861 2862 2863
    SArray *pWins = (SArray *) (*pIte);
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) {
        if (!pSeWin->isClosed) {
          pSeWin->isClosed = true;
          if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
2864
            int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed);
2865 2866
            pSeWin->isOutput = true;
          }
5
54liuyao 已提交
2867
        }
2868
        continue;
5
54liuyao 已提交
2869
      }
2870
      break;
5
54liuyao 已提交
2871 2872 2873 2874 2875
    }
  }
  return TSDB_CODE_SUCCESS;
}

2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887
int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) {
  void **pIte = NULL;
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    SArray *pWins = (SArray *) (*pIte);
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (!pSeWin->isClosed) {
        int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed);
        pSeWin->isOutput = true;
      }
5
54liuyao 已提交
2888 2889 2890 2891 2892
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
2893
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
5
54liuyao 已提交
2894 2895 2896 2897 2898
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2899
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
5
54liuyao 已提交
2900 2901 2902 2903 2904
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
2905
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
2906
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
5
54liuyao 已提交
2907 2908 2909 2910 2911
      doSetOperatorCompleted(pOperator);
    }
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
2912 2913
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
2914
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
2915
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
2916 2917 2918 2919 2920
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
2921

X
Xiaoyu Wang 已提交
2922 2923 2924
    if (pBlock->info.type == STREAM_REPROCESS) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      doClearSessionWindows(&pInfo->streamAggSup, &pInfo->binfo, pBlock, 0, pOperator->numOfExprs, pInfo->gap, pWins);
2925
      if (isFinalSession(pInfo)) {
X
Xiaoyu Wang 已提交
2926 2927
        int32_t                        childIndex = 0;  // Todo(liuyao) get child id from SSDataBlock
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
2928
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
X
Xiaoyu Wang 已提交
2929 2930
        doClearSessionWindows(&pChildInfo->streamAggSup, &pChildInfo->binfo, pBlock, 0, pChildOp->numOfExprs,
                              pChildInfo->gap, NULL);
5
54liuyao 已提交
2931
        rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->numOfExprs, pOperator->pTaskInfo);
2932 2933
      }
      taosArrayDestroy(pWins);
5
54liuyao 已提交
2934
      continue;
2935
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
2936 2937
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getSessionWinInfo);
      continue;
5
54liuyao 已提交
2938
    }
5
54liuyao 已提交
2939

2940 2941
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
2942
    if (isFinalSession(pInfo)) {
X
Xiaoyu Wang 已提交
2943
      int32_t         childIndex = 0;  // Todo(liuyao) get child id from SSDataBlock
2944
      SOptrBasicInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
5
54liuyao 已提交
2945
      doStreamSessionAggImpl(pOperator, pBlock, NULL, NULL);
2946
    }
5
54liuyao 已提交
2947
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted);
5
54liuyao 已提交
2948
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
2949 2950 2951
  }
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
2952

5
54liuyao 已提交
2953
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
X
Xiaoyu Wang 已提交
2954
                     getSessionWinInfo);
5
54liuyao 已提交
2955
  copyUpdateResult(pStUpdated, pUpdated);
5
54liuyao 已提交
2956
  taosHashCleanup(pStUpdated);
5
54liuyao 已提交
2957

5
54liuyao 已提交
2958
  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
X
Xiaoyu Wang 已提交
2959
                        pInfo->binfo.rowCellInfoOffset);
5
54liuyao 已提交
2960 2961 2962 2963 2964 2965
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
2966
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
2967 2968
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}
2969

2970 2971 2972 2973
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream,
    SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
  int32_t        code = TSDB_CODE_OUT_OF_MEMORY;
  SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo);
2974 2975 2976 2977
  if (pOperator == NULL) {
    goto _error;
  }
  pOperator->name = "StreamFinalSessionWindowAggOperator";
2978
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION;
2979
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2980
  pInfo->pChildren = taosArrayInit(8, sizeof(void*));
2981
  for (int32_t i = 0; i < numOfChild; i++) {
X
Xiaoyu Wang 已提交
2982
    SOperatorInfo* pChild =
2983
        createStreamSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo);
2984 2985 2986 2987 2988 2989 2990 2991 2992
    if (pChild == NULL) {
      goto _error;
    }
    taosArrayPush(pInfo->pChildren, &pChild);
  }
  return pOperator;

_error:
  if (pInfo != NULL) {
2993
    destroyStreamSessionAggOperatorInfo(pInfo, pOperator->numOfExprs);
2994 2995 2996 2997 2998 2999 3000
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5
54liuyao 已提交
3001 3002

void destroyStreamStateOperatorInfo(void* param, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
3003
  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;
5
54liuyao 已提交
3004 3005 3006 3007 3008 3009
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
3010
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
5
54liuyao 已提交
3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
      destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput);
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
}

int64_t getStateWinTsKey(void* data, int32_t index) {
  SStateWindowInfo* pStateWin = taosArrayGet(data, index);
  return pStateWin->winInfo.win.ekey;
}

X
Xiaoyu Wang 已提交
3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035
SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
3036 3037 3038 3039 3040 3041 3042 3043
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayPush(pWinInfos, &win);
}

X
Xiaoyu Wang 已提交
3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055
SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayInsert(pWinInfos, index, &win);
}

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {
  if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) {
    return true;
  }
  return false;
}

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {
  return pKeyData && compareVal(pKeyData, &pWin->stateKey);
}

3075 3076 3077
SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
X
Xiaoyu Wang 已提交
3078 3079
  int32_t           size = taosArrayGetSize(pWinInfos);
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index + 1;
      return pWin;
    }
  }
  *pIndex = 0;
  return NULL;
}

3100 3101 3102 3103
SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts,
    uint64_t groupId, char* pKeyData, SColumn* pCol, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
5
54liuyao 已提交
3104 3105 3106 3107 3108
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    *pIndex = 0;
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
X
Xiaoyu Wang 已提交
3109
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
  *pIndex = index + 1;
  return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol);
}

X
Xiaoyu Wang 已提交
3143 3144
int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, SColumnInfoData* pKeyCol, int32_t rows,
                              int32_t start, bool* allEqual, SHashObj* pSeDelete) {
5
54liuyao 已提交
3145 3146 3147 3148 3149
  *allEqual = true;
  SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex);
  for (int32_t i = start; i < rows; ++i) {
    char* pKeyData = colDataGetData(pKeyCol, i);
    if (!isTsInWindow(pWinInfo, pTs[i])) {
X
Xiaoyu Wang 已提交
3150
      if (isEqualStateKey(pWinInfo, pKeyData)) {
5
54liuyao 已提交
3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164
        int32_t size = taosArrayGetSize(pWinInfos);
        if (winIndex + 1 < size) {
          SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1);
          // ts belongs to the next window
          if (pTs[i] >= pNextWin->winInfo.win.skey) {
            return i - start;
          }
        }
      } else {
        return i - start;
      }
    }
    if (pWinInfo->winInfo.win.skey > pTs[i]) {
      if (pSeDelete && pWinInfo->winInfo.isOutput) {
X
Xiaoyu Wang 已提交
3165 3166
        taosHashPut(pSeDelete, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &pWinInfo->winInfo.win.skey,
                    sizeof(TSKEY));
5
54liuyao 已提交
3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179
        pWinInfo->winInfo.isOutput = false;
      }
      pWinInfo->winInfo.win.skey = pTs[i];
    }
    pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]);
    if (!isEqualStateKey(pWinInfo, pKeyData)) {
      *allEqual = false;
    }
  }
  return rows - start;
}

void deleteWindow(SArray* pWinInfos, int32_t index) {
X
Xiaoyu Wang 已提交
3180
  ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos));
5
54liuyao 已提交
3181 3182 3183
  taosArrayRemove(pWinInfos, index);
}

X
Xiaoyu Wang 已提交
3184 3185
static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int32_t tsIndex, SColumn* pCol,
                                int32_t keyIndex, SHashObj* pSeUpdated, SHashObj* pSeDeleted) {
5
54liuyao 已提交
3186 3187
  SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
  SColumnInfoData* pKeyColInfo = taosArrayGet(pBlock->pDataBlock, keyIndex);
X
Xiaoyu Wang 已提交
3188 3189 3190
  TSKEY*           tsCol = (TSKEY*)pTsColInfo->pData;
  bool             allEqual = false;
  int32_t          step = 1;
5
54liuyao 已提交
3191
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
3192 3193
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
3194
    SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], pBlock->info.groupId, &winIndex);
5
54liuyao 已提交
3195 3196 3197
    if (!pCurWin) {
      continue;
    }
3198
    step = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual,
X
Xiaoyu Wang 已提交
3199
                                 pSeDeleted);
5
54liuyao 已提交
3200 3201 3202
    ASSERT(isTsInWindow(pCurWin, tsCol[i]) || isEqualStateKey(pCurWin, pKeyData));
    taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win);
    taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
3203
    deleteWindow(pAggSup->pCurWins, winIndex);
5
54liuyao 已提交
3204 3205 3206
  }
}

X
Xiaoyu Wang 已提交
3207 3208 3209
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated,
                                 SHashObj* pStDeleted) {
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3210
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3211 3212 3213 3214 3215 3216 3217 3218 3219
  bool                         masterScan = true;
  int32_t                      numOfOutput = pOperator->numOfExprs;
  int64_t                      groupId = pSDataBlock->info.groupId;
  int64_t                      code = TSDB_CODE_SUCCESS;
  int32_t                      step = 1;
  bool                         ascScan = true;
  TSKEY*                       tsCols = NULL;
  SResultRow*                  pResult = NULL;
  int32_t                      winRows = 0;
5
54liuyao 已提交
3220
  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
3221 3222
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
3223
  } else {
X
Xiaoyu Wang 已提交
3224
    return;
5
54liuyao 已提交
3225
  }
X
Xiaoyu Wang 已提交
3226

5
54liuyao 已提交
3227
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
3228 3229 3230 3231 3232
  SColumnInfoData*     pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
  for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
    bool              allEqual = true;
3233 3234 3235 3236 3237
    SStateWindowInfo* pCurWin = 
        getStateWindow(pAggSup, tsCols[i], pSDataBlock->info.groupId, pKeyData,
            &pInfo->stateCol, &winIndex);
    winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, pKeyColInfo,
        pSDataBlock->info.rows, i, &allEqual, pInfo->pSeDeleted);
5
54liuyao 已提交
3238 3239 3240
    if (!allEqual) {
      taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win);
      taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
3241
      deleteWindow(pAggSup->pCurWins, winIndex);
5
54liuyao 已提交
3242 3243
      continue;
    }
X
Xiaoyu Wang 已提交
3244
    code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pTaskInfo);
5
54liuyao 已提交
3245 3246 3247 3248 3249
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    pCurWin->winInfo.isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
3250 3251 3252
      SWinRes value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId};
      code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition),
          &value, sizeof(SWinRes));
5
54liuyao 已提交
3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266
      if (code != TSDB_CODE_SUCCESS) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
      pCurWin->winInfo.isOutput = true;
    }
  }
}

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3267
  SOptrBasicInfo*              pBInfo = &pInfo->binfo;
5
54liuyao 已提交
3268 3269 3270 3271 3272
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
3273
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
3274
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
5
54liuyao 已提交
3275 3276 3277 3278 3279
      doSetOperatorCompleted(pOperator);
    }
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
3280 3281
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
3282
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
3283
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
3284 3285 3286 3287 3288
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
3289

5
54liuyao 已提交
3290
    if (pBlock->info.type == STREAM_REPROCESS) {
X
Xiaoyu Wang 已提交
3291 3292
      doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
                          pSeUpdated, pInfo->pSeDeleted);
5
54liuyao 已提交
3293
      continue;
3294
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
3295 3296
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getStateWinInfo);
      continue;
5
54liuyao 已提交
3297
    }
3298 3299 3300

    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
3301 3302 3303 3304 3305
    doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
X
Xiaoyu Wang 已提交
3306

5
54liuyao 已提交
3307
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
X
Xiaoyu Wang 已提交
3308
                     getStateWinInfo);
5
54liuyao 已提交
3309
  copyUpdateResult(pSeUpdated, pUpdated);
5
54liuyao 已提交
3310 3311 3312
  taosHashCleanup(pSeUpdated);

  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
X
Xiaoyu Wang 已提交
3313
                        pInfo->binfo.rowCellInfoOffset);
5
54liuyao 已提交
3314 3315 3316 3317 3318 3319
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
3320
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
3321 3322 3323
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

3324 3325 3326 3327
int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo));
}

X
Xiaoyu Wang 已提交
3328 3329 3330 3331 3332 3333
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
  SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
3334
  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
3335

X
Xiaoyu Wang 已提交
3336 3337
  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3338 3339 3340 3341
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

X
Xiaoyu Wang 已提交
3342
  int32_t    numOfCols = 0;
5
54liuyao 已提交
3343 3344 3345 3346
  SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
  initResultSizeInfo(pOperator, 4096);
3347
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
X
Xiaoyu Wang 已提交
3348 3349
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
5
54liuyao 已提交
3350 3351
      .calTrigger = pStateNode->window.triggerType,
      .maxTs = INT64_MIN,
X
Xiaoyu Wang 已提交
3352
  };
5
54liuyao 已提交
3353
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
3354 3355

  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
3356 3357 3358 3359
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

3360
  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pInfo->binfo.pCtx, numOfCols);
5
54liuyao 已提交
3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }

  initDummyFunction(pInfo->pDummyCtx, pInfo->binfo.pCtx, numOfCols);
  pInfo->primaryTsIndex = tsSlotId;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
  pInfo->pDelRes = createOneDataBlock(pResBlock, false);
  blockDataEnsureCapacity(pInfo->pDelRes, 64);
  pInfo->pChildren = NULL;

  pOperator->name = "StreamStateAggOperator";
3381
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
5
54liuyao 已提交
3382 3383 3384 3385 3386 3387
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->numOfExprs = numOfCols;
  pOperator->pExpr = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
3388 3389 3390
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL,
                                         destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType);
5
54liuyao 已提交
3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
  return pOperator;

_error:
  destroyStreamStateOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
3404 3405 3406 3407

typedef struct SMergeIntervalAggOperatorInfo {
  SIntervalAggOperatorInfo intervalAggOperatorInfo;

S
shenglian zhou 已提交
3408 3409 3410
  bool         hasGroupId;
  uint64_t     groupId;
  SSDataBlock* prefetchedBlock;
3411
  bool         inputBlocksFinished;
3412 3413 3414
} SMergeIntervalAggOperatorInfo;

void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) {
3415 3416 3417 3418
  SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
  destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput);
}

3419
static int32_t outputMergeIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock, TSKEY wstartTs) {
S
shenglian zhou 已提交
3420 3421 3422 3423
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
  bool                           ascScan = (iaInfo->order == TSDB_ORDER_ASC);
3424

3425 3426 3427 3428
  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId);
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf,
                                                            GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
  ASSERT(p1 != NULL);
3429

3430 3431 3432 3433
  finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, iaInfo->binfo.pCtx, pOperatorInfo->pExpr,
                                       pOperatorInfo->numOfExprs, iaInfo->binfo.rowCellInfoOffset, pResultBlock,
                                       pTaskInfo);
  taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
3434

3435
  return 0;
3436 3437 3438
}

static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
S
shenglian zhou 已提交
3439 3440 3441
                                   int32_t scanFlag, SSDataBlock* pResultBlock) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
3442 3443 3444 3445 3446

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

  int32_t     startPos = 0;
  int32_t     numOfOutput = pOperatorInfo->numOfExprs;
3447
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
3448 3449 3450 3451
  uint64_t    tableGroupId = pBlock->info.groupId;
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

3452 3453 3454
  STimeWindow win;
  win.skey = blockStartTs;
  win.ekey = taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1;
3455

3456
  //TODO: remove the hash table usage (groupid + winkey => result row position)
3457
  int32_t ret =
3458 3459
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, iaInfo->binfo.pCtx,
                             numOfOutput, iaInfo->binfo.rowCellInfoOffset, &iaInfo->aggSup, pTaskInfo);
3460 3461 3462 3463
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
  }

3464 3465 3466
  TSKEY currTs = blockStartTs;
  TSKEY currPos = startPos;
  STimeWindow currWin = win;
3467
  while (1) {
3468 3469
    ++currPos;
    if (currPos >= pBlock->info.rows) {
3470 3471
      break;
    }
3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489
    if (tsCols[currPos] == currTs) {
      continue;
    } else {
      updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
      doApplyFunctions(pTaskInfo, iaInfo->binfo.pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos,
                       currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->order);

      outputMergeIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs);

      currTs = tsCols[currPos];
      currWin.skey = currTs;
      currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1;
      startPos = currPos;
      ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, iaInfo->binfo.pCtx,
                                 numOfOutput, iaInfo->binfo.rowCellInfoOffset, &iaInfo->aggSup, pTaskInfo);
      if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
3490 3491
    }
  }
3492 3493 3494
  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
  doApplyFunctions(pTaskInfo, iaInfo->binfo.pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos,
                   currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->order);
3495

3496
  outputMergeIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs);
3497 3498 3499
}

static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
S
shenglian zhou 已提交
3500
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
3501 3502

  SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info;
S
shenglian zhou 已提交
3503
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
3504 3505 3506 3507
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

3508
  SSDataBlock* pRes = iaInfo->binfo.pRes;
3509
  blockDataCleanup(pRes);
3510
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);
3511

3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522
  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
        miaInfo->groupId = pBlock->info.groupId;
      }
3523

3524 3525 3526 3527
      if (pBlock == NULL) {
        miaInfo->inputBlocksFinished = true;
        break;
      }
3528

3529 3530 3531 3532 3533 3534 3535
      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
        miaInfo->groupId = pBlock->info.groupId;
      } else if (miaInfo->groupId != pBlock->info.groupId) {
        miaInfo->prefetchedBlock = pBlock;
        break;
      }
3536

3537 3538 3539
      getTableScanInfo(pOperator, &iaInfo->order, &scanFlag);
      setInputDataBlock(pOperator, iaInfo->binfo.pCtx, pBlock, iaInfo->order, scanFlag, true);
      doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);
3540

3541 3542 3543 3544 3545 3546
      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
        break;
      }
    }

    pRes->info.groupId = miaInfo->groupId;
3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558
  }

  if (pRes->info.rows == 0) {
    doSetOperatorCompleted(pOperator);
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
S
shenglian zhou 已提交
3559 3560
                                               SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
                                               SExecTaskInfo* pTaskInfo) {
3561
  SMergeIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
S
shenglian zhou 已提交
3562
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
3563 3564 3565 3566
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

S
shenglian zhou 已提交
3567
  SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
3568

3569 3570 3571 3572 3573 3574 3575
  iaInfo->win = pTaskInfo->window;
  iaInfo->order = TSDB_ORDER_ASC;
  iaInfo->interval = *pInterval;

  iaInfo->execModel = pTaskInfo->execModel;

  iaInfo->primaryTsIndex = primaryTsSlotId;
3576 3577 3578 3579 3580

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);

  int32_t code =
3581
      initAggInfo(&iaInfo->binfo, &iaInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
3582

3583
  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
3584

3585 3586 3587
  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(iaInfo->binfo.pCtx, numOfCols, iaInfo);
  if (iaInfo->timeWindowInterpo) {
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
3588 3589
  }

3590 3591
  //  iaInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo);
  if (code != TSDB_CODE_SUCCESS /* || iaInfo->pTableQueryInfo == NULL*/) {
3592 3593 3594
    goto _error;
  }

3595
  initResultRowInfo(&iaInfo->binfo.resultRowInfo);
3596 3597 3598 3599 3600 3601 3602 3603

  pOperator->name = "TimeMergeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL;
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->numOfExprs = numOfCols;
3604
  pOperator->info = miaInfo;
3605

3606
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL,
3607
                                         destroyMergeIntervalOperatorInfo, NULL, NULL, NULL);
3608 3609 3610 3611 3612 3613 3614 3615 3616

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
3617 3618
  destroyMergeIntervalOperatorInfo(miaInfo, numOfCols);
  taosMemoryFreeClear(miaInfo);
3619 3620 3621 3622
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}