timewindowoperator.c 138.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15
#include "executorimpl.h"
X
Xiaoyu Wang 已提交
16
#include "function.h"
5
54liuyao 已提交
17
#include "functionMgt.h"
L
Liu Jicong 已提交
18 19
#include "tdatablock.h"
#include "ttime.h"
H
Haojun Liao 已提交
20
#include "tfill.h"
21 22 23 24 25 26

typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;

5
54liuyao 已提交
27
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
5
54liuyao 已提交
28

29 30 31 32 33
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);

static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult);
static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult);

H
Haojun Liao 已提交
34 35 36 37 38 39 40 41 42 43 44 45
///*
// * There are two cases to handle:
// *
// * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including
// * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey.
// * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be
// *    merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there
// *    is a previous result generated or not.
// */
//static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) {
//  // do nothing
//}
46

X
Xiaoyu Wang 已提交
47
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69

static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w,
                                      bool ascQuery) {
  if (ascQuery) {
    getAlignQueryTimeWindow(pInterval, precision, ts, w);
  } else {
    // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
    getAlignQueryTimeWindow(pInterval, precision, ts, w);

    int64_t key = w->skey;
    while (key < ts) {  // moving towards end
      key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision);
      if (key >= ts) {
        break;
      }

      w->skey = key;
    }
  }
}

// get the correct time window according to the handled timestamp
X
Xiaoyu Wang 已提交
70 71
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
                                int32_t precision, STimeWindow* win) {
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
  STimeWindow w = {0};

  if (pResultRowInfo->cur.pageId == -1) {  // the first window, from the previous stored value
    getInitialStartTimeWindow(pInterval, precision, ts, &w, true);
    w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
  } else {
    w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win;
  }

  if (w.skey > ts || w.ekey < ts) {
    if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
      w.skey = taosTimeTruncate(ts, pInterval, precision);
      w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
    } else {
      int64_t st = w.skey;

      if (st > ts) {
        st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
      }

      int64_t et = st + pInterval->interval - 1;
      if (et < ts) {
        st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
      }

      w.skey = st;
      w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
    }
  }
  return w;
}

static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
                                      int32_t numOfOutput, int32_t* rowCellInfoOffset, SAggSupporter* pAggSup,
                                      SExecTaskInfo* pTaskInfo) {
  assert(win->skey <= win->ekey);
  SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                  masterscan, tableGroupId, pTaskInfo, true, pAggSup);

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // set time window for current result
  pResultRow->win = (*win);
119

120
  *pResult = pResultRow;
121
  setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowCellInfoOffset);
122

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
  return TSDB_CODE_SUCCESS;
}

static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) {
  int64_t* ts = (int64_t*)pColData->pData;
  int32_t  delta = includeEndpoint ? 1 : 0;

  int64_t duration = pWin->ekey - pWin->skey + delta;
  ts[2] = duration;            // set the duration
  ts[3] = pWin->skey;          // window start key
  ts[4] = pWin->ekey + delta;  // window end key
}

static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) {
  pRowSup->win.ekey = ts;
  pRowSup->prevTs = ts;
  pRowSup->numOfRows += 1;
}

static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex) {
  pRowSup->startRowIndex = rowIndex;
  pRowSup->numOfRows = 0;
  pRowSup->win.skey = tsList[rowIndex];
}

static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
                                                   int16_t pos, int16_t order, int64_t* pData) {
150
  int32_t forwardRows = 0;
151 152 153 154

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
155
      forwardRows = end;
156 157

      if (pData[end + pos] == ekey) {
158
        forwardRows += 1;
159 160 161
      }
    }
  } else {
162
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
163
    if (end >= 0) {
164
      forwardRows = end;
165

166
      if (pData[end + pos] == ekey) {
167
        forwardRows += 1;
168 169
      }
    }
X
Xiaoyu Wang 已提交
170 171 172 173 174 175 176 177
    //    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
    //    if (end >= 0) {
    //      forwardRows = pos - end;
    //
    //      if (pData[end] == ekey) {
    //        forwardRows += 1;
    //      }
    //    }
178 179
  }

180 181
  assert(forwardRows >= 0);
  return forwardRows;
182 183
}

5
54liuyao 已提交
184
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY*  keyList = (TSKEY*)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
201 202 203 204 205 206 207 208 209 210 211
      if (key >= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key < keyList[lastPos]) {
        lastPos += 1;
        if (lastPos >= num) {
          return -1;
        } else {
          return lastPos;
        }
      }
212 213 214 215 216 217

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        firstPos = midPos + 1;
218 219
      } else if (key > keyList[midPos]) {
        lastPos = midPos - 1;
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

X
Xiaoyu Wang 已提交
255 256
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
                                 __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num = -1;
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);

  if (order == TSDB_ORDER_ASC) {
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
      if (item != NULL) {
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
278
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
279 280
      }
    } else {
281
      num = pDataBlockInfo->rows - startPos;
282
      if (item != NULL) {
283
        item->lastKey = pDataBlockInfo->window.ekey + step;
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
      }
    }
  }

  assert(num >= 0);
  return num;
}

static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
    return;
  }

  int64_t key = tw->skey, interval = pInterval->interval;
  // convert key to second
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;

  if (pInterval->intervalUnit == 'y') {
    interval *= 12;
  }

  struct tm tm;
  time_t    t = (time_t)key;
  taosLocalTime(&t, &tm);

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision);

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision);

  tw->ekey -= 1;
}

X
Xiaoyu Wang 已提交
325
void doTimeWindowInterpolation(SIntervalAggOperatorInfo* pInfo, int32_t numOfExprs, SArray* pDataBlock, TSKEY prevTs,
326
                               int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) {
327
  SqlFunctionCtx* pCtx = pInfo->binfo.pCtx;
328

329 330
  int32_t index = 1;
  for (int32_t k = 0; k < numOfExprs; ++k) {
H
Haojun Liao 已提交
331
    if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
332 333 334 335
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

X
Xiaoyu Wang 已提交
336
    SFunctParam*     pParam = &pCtx[k].param[0];
337 338
    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId);

339
    ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey);
340

341
    double v1 = 0, v2 = 0, v = 0;
342
    if (prevRowIndex == -1) {
343 344
      SGroupKeys* p = taosArrayGet(pInfo->pPrevValues, index);
      GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData);
345
    } else {
346
      GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex));
347 348
    }

349
    GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
350

351
#if 0
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            //            pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
371 372
#endif

X
Xiaoyu Wang 已提交
373 374 375
    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
    SPoint point = (SPoint){.key = windowKey, .val = &v};
376

X
Xiaoyu Wang 已提交
377
    taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
378

X
Xiaoyu Wang 已提交
379 380 381 382 383 384
    if (type == RESULT_ROW_START_INTERP) {
      pCtx[k].start.key = point.key;
      pCtx[k].start.val = v;
    } else {
      pCtx[k].end.key = point.key;
      pCtx[k].end.val = v;
385
    }
X
Xiaoyu Wang 已提交
386 387 388

    index += 1;
  }
389
#if 0
390
  }
391
#endif
392 393 394 395 396 397 398 399 400 401 402 403 404 405
}

static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

X
Xiaoyu Wang 已提交
406 407 408
static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs,
                                              int32_t pos, SSDataBlock* pBlock, const TSKEY* tsCols, STimeWindow* win) {
  bool ascQuery = (pInfo->order == TSDB_ORDER_ASC);
409

410
  TSKEY curTs = tsCols[pos];
411 412

  SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
X
Xiaoyu Wang 已提交
413
  TSKEY       lastTs = *(int64_t*)pTsKey->pData;
414 415 416 417 418

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery ? win->skey : win->ekey;
  if (key == curTs) {
419
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP);
420 421 422
    return true;
  }

423 424
  // it is the first time window, no need to do interpolation
  if (pTsKey->isNull && pos == 0) {
425
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP);
426 427 428 429
  } else {
    TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]);
    doTimeWindowInterpolation(pInfo, numOfExprs, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key,
                              RESULT_ROW_START_INTERP);
430 431 432 433 434
  }

  return true;
}

X
Xiaoyu Wang 已提交
435 436 437
static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs,
                                            int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols,
                                            TSKEY blockEkey, STimeWindow* win) {
438
  int32_t order = pInfo->order;
439 440

  TSKEY actualEndKey = tsCols[endRowIndex];
441
  TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey;
442 443

  // not ended in current data block, do not invoke interpolation
444 445
  if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) {
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP);
446 447 448
    return false;
  }

449
  // there is actual end point of current time window, no interpolation needs
450
  if (key == actualEndKey) {
451
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP);
452 453 454
    return true;
  }

455
  int32_t nextRowIndex = endRowIndex + 1;
456 457 458
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
X
Xiaoyu Wang 已提交
459 460
  doTimeWindowInterpolation(pInfo, numOfExprs, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key,
                            RESULT_ROW_END_INTERP);
461 462 463 464
  return true;
}

static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
5
54liuyao 已提交
465
                                      TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
X
Xiaoyu Wang 已提交
466
  bool ascQuery = (order == TSDB_ORDER_ASC);
467 468 469 470 471 472 473 474 475 476

  int32_t precision = pInterval->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
    return -1;
  }

477
  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
478 479 480 481
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
482
    startPos = prevPosition + 1;
483
  } else {
484
    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
485 486
      startPos = 0;
    } else {
487
      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
    }
  }

  /* interp query with fill should not skip time window */
  //  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
  //    return startPos;
  //  }

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
    if (ascQuery) {
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
      }
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
      }
    }
  }

  return startPos;
}

531 532
static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
  ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp == true;
  }
}

static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp = true;
  }
}

X
Xiaoyu Wang 已提交
549 550 551
static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t numOfExprs,
                                        SqlFunctionCtx* pCtx, SResultRow* pResult, STimeWindow* win, int32_t startPos,
                                        int32_t forwardRows) {
552
  if (!pInfo->timeWindowInterpo) {
553 554 555
    return;
  }

556
  ASSERT(pBlock != NULL);
557 558 559 560 561
  if (pBlock->pDataBlock == NULL) {
    //    tscError("pBlock->pDataBlock == NULL");
    return;
  }

562
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
563 564

  TSKEY* tsCols = (TSKEY*)(pColInfo->pData);
565
  bool   done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
566
  if (!done) {  // it is not interpolated, now start to generated the interpolated value
567
    bool interp = setTimeWindowInterpolationStartTs(pInfo, pCtx, numOfExprs, startPos, pBlock, tsCols, win);
568 569 570 571
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
572
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP);
573 574 575 576 577 578 579 580
  }

  // point interpolation does not require the end key time window interpolation.
  //  if (pointInterpQuery) {
  //    return;
  //  }

  // interpolation query does not generate the time window end interpolation
581
  done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
582
  if (!done) {
583
    int32_t endRowIndex = startPos + forwardRows - 1;
584

585
    TSKEY endKey = (pInfo->order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
586
    bool  interp =
587
        setTimeWindowInterpolationEndTs(pInfo, pCtx, numOfExprs, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
588 589 590 591
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
592
    setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP);
593 594 595
  }
}

596 597
static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) {
  if (pBlock->pDataBlock == NULL) {
598 599 600
    return;
  }

601 602 603 604 605 606 607
  size_t num = taosArrayGetSize(pPrevKeys);
  for (int32_t k = 0; k < num; ++k) {
    SColumn* pc = taosArrayGet(pCols, k);

    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId);

    SGroupKeys* pkey = taosArrayGet(pPrevKeys, k);
X
Xiaoyu Wang 已提交
608
    for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) {
609 610 611 612 613 614 615 616 617 618 619 620 621 622
      if (colDataIsNull_s(pColInfo, i)) {
        continue;
      }

      char* val = colDataGetData(pColInfo, i);
      if (IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, varDataTLen(val));
        ASSERT(varDataTLen(val) <= pkey->bytes);
      } else {
        memcpy(pkey->pData, val, pkey->bytes);
      }

      break;
    }
623 624 625
  }
}

626 627 628 629
static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo,
                                       SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) {
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

630
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
631

632 633 634
  int32_t  startPos = 0;
  int32_t  numOfOutput = pOperatorInfo->numOfExprs;
  uint64_t groupId = pBlock->info.groupId;
635

636
  SResultRow* pResult = NULL;
637

638 639
  while (1) {
    SListNode* pn = tdListGetHead(pResultRowInfo->openWindow);
640

641 642 643 644
    SResultRowPosition* p1 = (SResultRowPosition*)pn->data;
    if (p->pageId == p1->pageId && p->offset == p1->offset) {
      break;
    }
645

646 647
    SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1);
    ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
648

649
    if (pr->closed) {
X
Xiaoyu Wang 已提交
650 651
      ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
             isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
652 653 654
      tdListPopHead(pResultRowInfo->openWindow);
      continue;
    }
655

656
    STimeWindow w = pr->win;
X
Xiaoyu Wang 已提交
657 658 659
    int32_t     ret =
        setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pInfo->binfo.pCtx,
                               numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
660 661 662 663 664 665
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP));

X
Xiaoyu Wang 已提交
666 667 668 669
    SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
    int64_t     prevTs = *(int64_t*)pTsKey->pData;
    doTimeWindowInterpolation(pInfo, numOfOutput, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey,
                              RESULT_ROW_END_INTERP);
670 671 672 673 674 675 676 677 678 679

    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    setNotInterpoWindowKey(pInfo->binfo.pCtx, numOfExprs, RESULT_ROW_START_INTERP);

    doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &w, &pInfo->twAggSup.timeWindowData, startPos, 0, tsCols,
                     pBlock->info.rows, numOfExprs, pInfo->order);

    if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
      closeResultRow(pr);
      tdListPopHead(pResultRowInfo->openWindow);
X
Xiaoyu Wang 已提交
680
    } else {  // the remains are can not be closed yet.
681
      break;
682
    }
683
  }
684
}
685

5
54liuyao 已提交
686
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
687

X
Xiaoyu Wang 已提交
688 689 690
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;
5
54liuyao 已提交
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (key >= getValuefn(keyList, lastPos)) return lastPos;
      if (key == getValuefn(keyList, firstPos)) return firstPos;
      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (key <= getValuefn(keyList, firstPos)) return firstPos;
      if (key == getValuefn(keyList, lastPos)) return lastPos;

      if (key > getValuefn(keyList, lastPos)) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
737 738
  }

5
54liuyao 已提交
739 740 741 742
  return midPos;
}

int64_t getReskey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
743
  SArray*     res = (SArray*)data;
5
54liuyao 已提交
744 745 746 747
  SResKeyPos* pos = taosArrayGetP(res, index);
  return *(int64_t*)pos->key;
}

5
54liuyao 已提交
748 749
static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId,
    SArray* pUpdated) {
5
54liuyao 已提交
750
  int32_t size = taosArrayGetSize(pUpdated);
5
54liuyao 已提交
751
  int32_t index = binarySearch(pUpdated, size, ts, TSDB_ORDER_DESC, getReskey);
5
54liuyao 已提交
752 753 754 755
  if (index == -1) {
    index = 0;
  } else {
    TSKEY resTs = getReskey(pUpdated, index);
5
54liuyao 已提交
756
    if (resTs < ts) {
5
54liuyao 已提交
757 758 759 760 761
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
H
Haojun Liao 已提交
762

5
54liuyao 已提交
763 764 765 766 767
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
5
54liuyao 已提交
768 769
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
X
Xiaoyu Wang 已提交
770
  if (taosArrayInsert(pUpdated, index, &newPos) == NULL) {
5
54liuyao 已提交
771 772 773 774 775
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
776 777 778 779
static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
  return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated);
}

5
54liuyao 已提交
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795
static void removeResult(SArray* pUpdated, TSKEY key) {
  int32_t size = taosArrayGetSize(pUpdated);
  int32_t index = binarySearch(pUpdated, size, key, TSDB_ORDER_DESC, getReskey);
  if (index >= 0 && key == getReskey(pUpdated, index)) {
    taosArrayRemove(pUpdated, index);
  }
}

static void removeResults(SArray* pWins, SArray* pUpdated) {
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
    STimeWindow* pW = taosArrayGet(pWins, i);
    removeResult(pUpdated, pW->skey);
  }
}

5
54liuyao 已提交
796
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
797
                            int32_t scanFlag, SArray* pUpdated) {
798
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
799

800 801
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

X
Xiaoyu Wang 已提交
802 803 804 805 806 807 808
  int32_t     startPos = 0;
  int32_t     numOfOutput = pOperatorInfo->numOfExprs;
  int64_t*    tsCols = extractTsCol(pBlock, pInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
  bool        ascScan = (pInfo->order == TSDB_ORDER_ASC);
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;
809 810 811 812

  STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
                                        pInfo->interval.precision, &pInfo->win);

X
Xiaoyu Wang 已提交
813 814 815
  int32_t ret =
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx,
                             numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
816 817 818 819
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
  }

5
54liuyao 已提交
820
  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
821
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
822
      saveResultRow(pResult, tableGroupId, pUpdated);
5
54liuyao 已提交
823
    }
824 825
  }

X
Xiaoyu Wang 已提交
826 827 828
  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
829
  ASSERT(forwardRows > 0);
830 831

  // prev time window not interpolation yet.
832
  if (pInfo->timeWindowInterpo) {
833 834
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult);
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);
835 836

    // restore current time window
X
Xiaoyu Wang 已提交
837 838 839
    ret =
        setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx,
                               numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
840 841 842 843
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

844
    // window start key interpolation
845
    doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &win, startPos, forwardRows);
846
  }
847 848

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
849 850 851 852
  doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
                   pBlock->info.rows, numOfOutput, pInfo->order);

  doCloseWindow(pResultRowInfo, pInfo, pResult);
853 854 855

  STimeWindow nextWin = win;
  while (1) {
856
    int32_t prevEndPos = forwardRows - 1 + startPos;
857
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->order);
858 859 860 861 862
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
X
Xiaoyu Wang 已提交
863 864 865
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                          pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset,
                                          &pInfo->aggSup, pTaskInfo);
866 867 868 869
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

5
54liuyao 已提交
870
    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
871
      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
872
        saveResultRow(pResult, tableGroupId, pUpdated);
5
54liuyao 已提交
873
      }
874 875
    }

X
Xiaoyu Wang 已提交
876
    ekey = ascScan ? nextWin.ekey : nextWin.skey;
877
    forwardRows =
878
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
879 880

    // window start(end) key interpolation
X
Xiaoyu Wang 已提交
881 882
    doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos,
                                forwardRows);
883 884

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
X
Xiaoyu Wang 已提交
885 886
    doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     tsCols, pBlock->info.rows, numOfOutput, pInfo->order);
887
    doCloseWindow(pResultRowInfo, pInfo, pResult);
888 889 890
  }

  if (pInfo->timeWindowInterpo) {
891
    saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
892
  }
893 894 895 896 897 898 899 900 901 902 903 904
}

void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
  // current result is done in computing final results.
  if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
    closeResultRow(pResult);
    tdListPopHead(pResultRowInfo->openWindow);
  }
}

SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult) {
  SResultRowPosition pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
X
Xiaoyu Wang 已提交
905
  SListNode*         pn = tdListGetTail(pResultRowInfo->openWindow);
906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930
  if (pn == NULL) {
    tdListAppend(pResultRowInfo->openWindow, &pos);
    return pos;
  }

  SResultRowPosition* px = (SResultRowPosition*)pn->data;
  if (px->pageId != pos.pageId || px->offset != pos.offset) {
    tdListAppend(pResultRowInfo->openWindow, &pos);
  }

  return pos;
}

int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
  TSKEY* tsCols = NULL;
  if (pBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;

    if (tsCols != NULL) {
      blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
    }
  }

  return tsCols;
931 932 933 934 935 936 937
}

static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
  }

L
Liu Jicong 已提交
938
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
939
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
940

941 942
  int32_t scanFlag = MAIN_SCAN;

X
Xiaoyu Wang 已提交
943
  int64_t        st = taosGetTimestampUs();
944 945 946
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
947
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
948 949 950 951
    if (pBlock == NULL) {
      break;
    }

952 953
    getTableScanInfo(pOperator, &pInfo->order, &scanFlag);

954
    // the pDataBlock are always the same one, no need to call this again
955
    setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, scanFlag, true);
H
Haojun Liao 已提交
956
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL);
957 958

#if 0  // test for encode/decode result info
959
    if(pOperator->fpSet.encodeResultRow){
960 961 962
      char *result = NULL;
      int32_t length = 0;
      SAggSupporter   *pSup = &pInfo->aggSup;
963
      pOperator->fpSet.encodeResultRow(pOperator, &result, &length);
964 965
      taosHashClear(pSup->pResultRowHashTable);
      pInfo->binfo.resultRowInfo.size = 0;
966
      pOperator->fpSet.decodeResultRow(pOperator, result);
967 968 969 970 971 972 973 974
      if(result){
        taosMemoryFree(result);
      }
    }
#endif
  }

  closeAllResultRows(&pInfo->binfo.resultRowInfo);
975
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->order);
976
  OPTR_SET_OPENED(pOperator);
977 978

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
979 980 981
  return TSDB_CODE_SUCCESS;
}

982 983 984 985 986 987 988 989 990 991 992 993
static bool compareVal(const char* v, const SStateKeys* pKey) {
  if (IS_VAR_DATA_TYPE(pKey->type)) {
    if (varDataLen(v) != varDataLen(pKey->pData)) {
      return false;
    } else {
      return strncmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0;
    }
  } else {
    return memcmp(pKey->pData, v, pKey->bytes) == 0;
  }
}

994
static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
L
Liu Jicong 已提交
995
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
996

997
  SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
998 999 1000
  int64_t          gid = pBlock->info.groupId;

  bool    masterScan = true;
1001
  int32_t numOfOutput = pOperator->numOfExprs;
1002 1003
  int16_t bytes = pStateColInfoData->info.bytes;

1004
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1005 1006 1007 1008 1009
  TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

1010
  struct SColumnDataAgg* pAgg = NULL;
1011
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
X
Xiaoyu Wang 已提交
1012
    pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
1013
    if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
1014 1015 1016 1017 1018 1019
      continue;
    }

    char* val = colDataGetData(pStateColInfoData, j);

    if (!pInfo->hasKey) {
1020 1021 1022 1023 1024 1025 1026
      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }

1027 1028 1029 1030
      pInfo->hasKey = true;

      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
1031
    } else if (compareVal(val, &pInfo->stateKey)) {
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
      doKeepTuple(pRowSup, tsList[j]);
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // a new state window started
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
      int32_t ret =
          setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pInfo->binfo.pCtx,
                                 numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
      }

      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1051
      doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1052 1053 1054 1055 1056
                       pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);

      // here we start a new session window
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
1057 1058 1059 1060 1061 1062 1063

      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
  int32_t ret =
      setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pInfo->binfo.pCtx,
                             numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
1077
  doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1078 1079 1080
                   pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
}

1081
static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
1082 1083 1084 1085 1086
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
1087 1088 1089

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
  SOptrBasicInfo* pBInfo = &pInfo->binfo;
1090 1091

  if (pOperator->status == OP_RES_TO_RETURN) {
1092
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1093
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1094 1095 1096 1097 1098 1099 1100
      doSetOperatorCompleted(pOperator);
      return NULL;
    }

    return pBInfo->pRes;
  }

1101
  int32_t order = TSDB_ORDER_ASC;
1102
  int64_t st = taosGetTimestampUs();
1103 1104 1105

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  while (1) {
1106
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1107 1108 1109 1110
    if (pBlock == NULL) {
      break;
    }

1111
    setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, MAIN_SCAN, true);
1112 1113
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1114 1115 1116
    doStateWindowAggImpl(pOperator, pInfo, pBlock);
  }

X
Xiaoyu Wang 已提交
1117
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1118

1119 1120 1121
  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pBInfo->resultRowInfo);

1122
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1123
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1124
  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1125
  if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1126 1127 1128
    doSetOperatorCompleted(pOperator);
  }

1129 1130 1131
  size_t rows = pBInfo->pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1132
  return (rows == 0) ? NULL : pBInfo->pRes;
1133 1134
}

1135
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
1136
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
1137
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1138 1139 1140 1141 1142 1143 1144 1145

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pBlock = pInfo->binfo.pRes;

  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
1146
    return pOperator->fpSet.getStreamResFn(pOperator);
1147 1148 1149 1150 1151 1152 1153
  } else {
    pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
    if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
      return NULL;
    }

    blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
1154
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1155

1156
    if (pBlock->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1157 1158 1159
      doSetOperatorCompleted(pOperator);
    }

1160 1161 1162
    size_t rows = pBlock->info.rows;
    pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1163
    return (rows == 0) ? NULL : pBlock;
1164 1165 1166 1167
  }
}

// todo merged with the build group result.
1168
static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
1169 1170 1171 1172 1173 1174 1175 1176
                                  int32_t* rowCellInfoOffset) {
  size_t num = taosArrayGetSize(pUpdateList);

  for (int32_t i = 0; i < num; ++i) {
    SResKeyPos* pPos = taosArrayGetP(pUpdateList, i);

    SFilePage*  bufPage = getBufPage(pBuf, pPos->pos.pageId);
    SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
1177

1178
    for (int32_t j = 0; j < numOfOutput; ++j) {
1179
      SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, j, rowCellInfoOffset);
1180 1181
      if (pRow->numOfRows < pEntry->numOfRes) {
        pRow->numOfRows = pEntry->numOfRes;
1182 1183 1184 1185 1186 1187
      }
    }

    releaseBufPage(pBuf, bufPage);
  }
}
5
54liuyao 已提交
1188
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
L
Liu Jicong 已提交
1189
  for (int i = 0; i < num; i++) {
5
54liuyao 已提交
1190 1191
    if (type == STREAM_INVERT) {
      fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
L
Liu Jicong 已提交
1192
    } else if (type == STREAM_NORMAL) {
5
54liuyao 已提交
1193 1194 1195 1196
      fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
    }
  }
}
5
54liuyao 已提交
1197

1198
void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SOptrBasicInfo* pBinfo, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
1199
  SResultRow*     pResult = getResultRowByPos(pResultBuf, p1);
5
54liuyao 已提交
1200 1201
  SqlFunctionCtx* pCtx = pBinfo->pCtx;
  for (int32_t i = 0; i < numOfOutput; ++i) {
1202
    pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pBinfo->rowCellInfoOffset);
5
54liuyao 已提交
1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
      continue;
    }
    pResInfo->initialized = false;
    if (pCtx[i].functionId != -1) {
      pCtx[i].fpSet.init(&pCtx[i], pResInfo);
    }
  }
}

X
Xiaoyu Wang 已提交
1214 1215
void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, int16_t bytes, uint64_t groupId,
                   int32_t numOfOutput) {
5
54liuyao 已提交
1216 1217
  SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId);
  SResultRowPosition* p1 =
X
Xiaoyu Wang 已提交
1218
      (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1219 1220 1221 1222
  if (!p1) {
    // window has been closed
    return;
  }
5
54liuyao 已提交
1223 1224 1225
  doClearWindowImpl(p1, pSup->pResultBuf, pBinfo, numOfOutput);
}

X
Xiaoyu Wang 已提交
1226 1227
static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, SInterval* pInterval, int32_t tsIndex,
                           int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) {
5
54liuyao 已提交
1228
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
1229 1230
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
  int32_t          step = 0;
5
54liuyao 已提交
1231 1232 1233
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
X
Xiaoyu Wang 已提交
1234 1235
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL);
    step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
5
54liuyao 已提交
1236
    doClearWindow(pSup, pBinfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput);
1237 1238 1239
    if (pUpWins) {
      taosArrayPush(pUpWins, &win);
    }
5
54liuyao 已提交
1240 1241
  }
}
1242

5
54liuyao 已提交
1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259
static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) {
  void*  pIte = NULL;
  size_t keyLen = 0;
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    void*    key = taosHashGetKey(pIte, &keyLen);
    uint64_t groupId = *(uint64_t*)key;
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
    TSKEY          ts = *(int64_t*)((char*)key + sizeof(uint64_t));
    SResultRowPosition* pPos = (SResultRowPosition*)pIte;
    int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
1260 1261 1262 1263
bool isCloseWindow(STimeWindow *pWin, STimeWindowAggSupp* pSup) {
  return pWin->ekey < pSup->maxTs - pSup->waterMark;
}

X
Xiaoyu Wang 已提交
1264 1265 1266
static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
                                   SArray* closeWins) {
  void*  pIte = NULL;
5
54liuyao 已提交
1267
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
1268 1269 1270
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    void*    key = taosHashGetKey(pIte, &keyLen);
    uint64_t groupId = *(uint64_t*)key;
5
54liuyao 已提交
1271
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
X
Xiaoyu Wang 已提交
1272
    TSKEY          ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1273 1274
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
X
Xiaoyu Wang 已提交
1275
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL);
5
54liuyao 已提交
1276
    if (isCloseWindow(&win, pSup)) {
5
54liuyao 已提交
1277 1278
      char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
      SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
1279
      taosHashRemove(pHashMap, keyBuf, keyLen);
5
54liuyao 已提交
1280 1281 1282 1283 1284 1285
      SResultRowPosition* pPos = (SResultRowPosition*)pIte;
      if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
        int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins);
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
5
54liuyao 已提交
1286 1287 1288 1289 1290 1291
      }
    }
  }
  return TSDB_CODE_SUCCESS;
}

1292
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
1293
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
1294
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1295 1296

  pInfo->order = TSDB_ORDER_ASC;
1297 1298 1299 1300 1301 1302

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
1303
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1304
    if (pInfo->binfo.pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1305 1306 1307 1308 1309 1310 1311
      pOperator->status = OP_EXEC_DONE;
    }
    return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];

5
54liuyao 已提交
1312
  SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);
1313
  while (1) {
1314
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1315 1316 1317 1318
    if (pBlock == NULL) {
      break;
    }

5
54liuyao 已提交
1319
    if (pBlock->info.type == STREAM_REPROCESS) {
X
Xiaoyu Wang 已提交
1320
      doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, 0, pOperator->numOfExprs, pBlock, NULL);
1321
      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
5
54liuyao 已提交
1322
      continue;
1323
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
1324 1325
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
      continue;
5
54liuyao 已提交
1326
    }
1327

1328 1329 1330 1331 1332 1333 1334 1335
    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
    // caller. Note that all the time window are not close till now.
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
    if (pInfo->invertible) {
      setInverFunction(pInfo->binfo.pCtx, pOperator->numOfExprs, pBlock->info.type);
    }

5
54liuyao 已提交
1336
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
H
Haojun Liao 已提交
1337
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated);
1338
  }
5
54liuyao 已提交
1339
  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
1340

1341
  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset);
1342 1343
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
1344
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357

  pOperator->status = OP_RES_TO_RETURN;

  return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}

static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  taosMemoryFreeClear(pInfo->stateKey.pData);
}

void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) {
1358
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
1359 1360 1361 1362
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  cleanupAggSup(&pInfo->aggSup);
}

5
54liuyao 已提交
1363
void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
1364
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
5
54liuyao 已提交
1365 1366
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  cleanupAggSup(&pInfo->aggSup);
1367 1368 1369 1370 1371 1372 1373 1374 1375
  if (pInfo->pChildren) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
      destroyIntervalOperatorInfo(pChildOp->info, numOfOutput);
      taosMemoryFreeClear(pChildOp->info);
      taosMemoryFreeClear(pChildOp);
    }
  }
1376
  nodesDestroyNode((SNode*)pInfo->pPhyNode);
5
54liuyao 已提交
1377 1378
}

1379
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
5
54liuyao 已提交
1380 1381 1382 1383 1384 1385 1386 1387
  for (int32_t i = 0; i < numOfCols; i++) {
    if (!fmIsInvertible(pFCtx[i].functionId)) {
      return false;
    }
  }
  return true;
}

1388
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
1389 1390
  // the primary timestamp column
  bool needed = false;
1391 1392
  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
1393

X
Xiaoyu Wang 已提交
1394
  {  // ts column
1395 1396
    SColumn c = {0};
    c.colId = 1;
1397
    c.slotId = pInfo->primaryTsIndex;
1398 1399
    c.type = TSDB_DATA_TYPE_TIMESTAMP;
    c.bytes = sizeof(int64_t);
1400
    taosArrayPush(pInfo->pInterpCols, &c);
1401 1402

    SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1403 1404 1405 1406
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = true;  // to denote no value is assigned yet
    key.pData = taosMemoryCalloc(1, c.bytes);
1407
    taosArrayPush(pInfo->pPrevValues, &key);
1408 1409
  }

X
Xiaoyu Wang 已提交
1410
  for (int32_t i = 0; i < numOfCols; ++i) {
1411 1412
    SExprInfo* pExpr = pCtx[i].pExpr;

H
Haojun Liao 已提交
1413
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1414 1415 1416
      SFunctParam* pParam = &pExpr->base.pParam[0];

      SColumn c = *pParam->pCol;
1417
      taosArrayPush(pInfo->pInterpCols, &c);
1418 1419 1420
      needed = true;

      SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1421 1422
      key.bytes = c.bytes;
      key.type = c.type;
1423
      key.isNull = false;
X
Xiaoyu Wang 已提交
1424
      key.pData = taosMemoryCalloc(1, c.bytes);
1425
      taosArrayPush(pInfo->pPrevValues, &key);
1426 1427 1428 1429 1430 1431
    }
  }

  return needed;
}

1432 1433 1434 1435 1436 1437
void increaseTs(SqlFunctionCtx* pCtx) {
  if (pCtx[0].pExpr->pExpr->_function.pFunctNode->funcType == FUNCTION_TYPE_WSTARTTS) {
    pCtx[0].increase = true;
  }
}

1438 1439
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                          SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
1440
                                          STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream) {
1441
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1442
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1443 1444 1445 1446
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

X
Xiaoyu Wang 已提交
1447 1448 1449
  pInfo->win = pTaskInfo->window;
  pInfo->order = TSDB_ORDER_ASC;
  pInfo->interval = *pInterval;
L
Liu Jicong 已提交
1450
  pInfo->execModel = pTaskInfo->execModel;
X
Xiaoyu Wang 已提交
1451
  pInfo->twAggSup = *pTwAggSupp;
1452

1453 1454
  pInfo->primaryTsIndex = primaryTsSlotId;

1455 1456
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);
1457 1458 1459

  int32_t code =
      initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
1460 1461 1462 1463 1464
  
  if (isStream) {
    ASSERT(numOfCols > 0);
    increaseTs(pInfo->binfo.pCtx);
  }
1465

1466
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
1467

5
54liuyao 已提交
1468
  pInfo->invertible = allInvertible(pInfo->binfo.pCtx, numOfCols);
X
Xiaoyu Wang 已提交
1469
  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
1470

1471
  pInfo->timeWindowInterpo = timeWindowinterpNeeded(pInfo->binfo.pCtx, numOfCols, pInfo);
1472 1473
  if (pInfo->timeWindowInterpo) {
    pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
H
Haojun Liao 已提交
1474 1475 1476
    if (pInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
1477 1478
  }

1479
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1480

X
Xiaoyu Wang 已提交
1481 1482 1483 1484 1485 1486 1487 1488
  pOperator->name = "TimeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->numOfExprs = numOfCols;
  pOperator->info = pInfo;
1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499

  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL,
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1500
_error:
1501 1502 1503 1504 1505 1506 1507 1508 1509
  destroyIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                                SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
wmmhello's avatar
wmmhello 已提交
1510
                                                STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
1511
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1512
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

  pInfo->order = TSDB_ORDER_ASC;
  pInfo->interval = *pInterval;
  pInfo->execModel = OPTR_EXEC_MODEL_STREAM;
  pInfo->win = pTaskInfo->window;
  pInfo->twAggSup = *pTwAggSupp;
  pInfo->primaryTsIndex = primaryTsSlotId;

  int32_t numOfRows = 4096;
  size_t  keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

  initResultSizeInfo(pOperator, numOfRows);
  int32_t code =
      initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);

wmmhello's avatar
wmmhello 已提交
1532
  if (code != TSDB_CODE_SUCCESS) {
1533 1534 1535
    goto _error;
  }

1536
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1537 1538

  pOperator->name = "StreamTimeIntervalAggOperator";
X
Xiaoyu Wang 已提交
1539
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
1540
  pOperator->blocking = true;
1541 1542 1543
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
1544
  pOperator->numOfExprs = numOfCols;
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556
  pOperator->info = pInfo;

  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL,
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1557
_error:
1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568
  destroyIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

// todo handle multiple tables cases.
static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

1569
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1570 1571

  bool    masterScan = true;
1572
  int32_t numOfOutput = pOperator->numOfExprs;
1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
  int64_t gid = pBlock->info.groupId;

  int64_t gap = pInfo->gap;

  if (!pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->winSup.prevTs = INT64_MIN;
  }

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

  // In case of ascending or descending order scan data, only one time window needs to be kepted for each table.
  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
    if (pInfo->winSup.prevTs == INT64_MIN) {
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
    } else if (tsList[j] - pRowSup->prevTs <= gap && (tsList[j] - pRowSup->prevTs) >= 0) {
      // The gap is less than the threshold, so it belongs to current session window that has been opened already.
      doKeepTuple(pRowSup, tsList[j]);
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
      int32_t ret =
          setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pInfo->binfo.pCtx,
                                 numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
      }

      // pInfo->numOfRows data belong to the current session window
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1613
      doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631
                       pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);

      // here we start a new session window
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
  int32_t ret =
      setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pInfo->binfo.pCtx,
                             numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
1632
  doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1633 1634 1635
                   pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
}

1636
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
1637 1638 1639 1640 1641 1642 1643 1644
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*          pBInfo = &pInfo->binfo;

  if (pOperator->status == OP_RES_TO_RETURN) {
1645
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1646
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1647 1648 1649 1650 1651 1652 1653
      doSetOperatorCompleted(pOperator);
      return NULL;
    }

    return pBInfo->pRes;
  }

1654 1655 1656
  int64_t st = taosGetTimestampUs();
  int32_t order = TSDB_ORDER_ASC;

1657 1658 1659
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1660
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1661 1662 1663 1664 1665
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1666
    setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, MAIN_SCAN, true);
1667 1668
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1669 1670 1671
    doSessionWindowAggImpl(pOperator, pInfo, pBlock);
  }

1672 1673
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;

1674 1675 1676 1677
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pBInfo->resultRowInfo);

1678
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1679
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1680
  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1681
  if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1682 1683 1684
    doSetOperatorCompleted(pOperator);
  }

1685 1686 1687
  size_t rows = pBInfo->pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1688
  return (rows == 0) ? NULL : pBInfo->pRes;
1689 1690
}

H
Haojun Liao 已提交
1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock) {
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for(int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData*  pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);

      pkey->isNull = false;
      char* val = colDataGetData(pColInfoData, i);
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }
}

H
Haojun Liao 已提交
1707
static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
1708 1709 1710 1711 1712
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
H
Haojun Liao 已提交
1713 1714
  SSDataBlock* pResBlock = pSliceInfo->binfo.pRes;

H
Haojun Liao 已提交
1715 1716
//  if (pOperator->status == OP_RES_TO_RETURN) {
//    //    doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes);
1717
//    if (pResBlock->info.rows == 0 || !hasDataInGroupInfo(&pSliceInfo->groupResInfo)) {
H
Haojun Liao 已提交
1718 1719 1720 1721 1722
//      doSetOperatorCompleted(pOperator);
//    }
//
//    return pResBlock;
//  }
1723

H
Haojun Liao 已提交
1724 1725
  int32_t order = TSDB_ORDER_ASC;
  SInterval* pInterval = &pSliceInfo->interval;
1726 1727
  SOperatorInfo* downstream = pOperator->pDownstream[0];

H
Haojun Liao 已提交
1728
  int32_t numOfRows = 0;
1729
  while (1) {
1730
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1731 1732 1733 1734 1735
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1736
    setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
H
Haojun Liao 已提交
1737 1738 1739 1740 1741 1742

    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, 0);
    for(int32_t i = 0; i < pBlock->info.rows; ++i) {
      int64_t ts = *(int64_t*) colDataGetData(pTsCol, i);

      if (ts == pSliceInfo->current) {
H
Haojun Liao 已提交
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755
        for(int32_t j = 0; j < pOperator->numOfExprs; ++j) {
          SExprInfo* pExprInfo = &pOperator->pExpr[j];
          int32_t dstSlot = pExprInfo->base.resSchema.slotId;
          int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;

          SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
          SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, dstSlot);

          char* v = colDataGetData(pSrc, i);
          colDataAppend(pDst, numOfRows, v, false);
        }

        numOfRows += 1;
H
Haojun Liao 已提交
1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766

        pSliceInfo->current += taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
        if (pSliceInfo->current > pSliceInfo->win.ekey) {
          doSetOperatorCompleted(pOperator);
          break;
        }
      } else if (ts < pSliceInfo->current) {
        if (i != pBlock->info.window.ekey) {
          int64_t nextTs = *(int64_t*) colDataGetData(pTsCol, i + 1);
          if (nextTs > pSliceInfo->current) {
            // output the result
H
Haojun Liao 已提交
1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850
            for (int32_t j = 0; j < pOperator->numOfExprs; ++j) {
              SExprInfo* pExprInfo = &pOperator->pExpr[j];
              int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
              int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;

              SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
              SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, dstSlot);

              switch (pSliceInfo->fillType) {
                case TSDB_FILL_NULL:
                  colDataAppendNULL(pDst, numOfRows);
                  break;

                case TSDB_FILL_SET_VALUE: {
                  SVariant* pVar = &pSliceInfo->pFillColInfo[i].fillVal;

                  if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
                    float v = 0;
                    GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
                    colDataAppend(pDst, numOfRows, (char*)&v, false);
                  } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
                    double v = 0;
                    GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
                    colDataAppend(pDst, numOfRows, (char*)&v, false);
                  } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
                    int64_t v = 0;
                    GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
                    colDataAppend(pDst, numOfRows, (char*)&v, false);
                  }
                }
                break;

                case TSDB_FILL_LINEAR:
#if 0
                if (pCtx->start.key == INT64_MIN || pCtx->start.key > pCtx->startTs
                    || pCtx->end.key == INT64_MIN || pCtx->end.key < pCtx->startTs) {
//                  goto interp_exit;
                }

              double v1 = -1, v2 = -1;
              GET_TYPED_DATA(v1, double, pCtx->inputType, &pCtx->start.val);
              GET_TYPED_DATA(v2, double, pCtx->inputType, &pCtx->end.val);

              SPoint point1 = {.key = ts, .val = &v1};
              SPoint point2 = {.key = nextTs, .val = &v2};
              SPoint point  = {.key = pCtx->startTs, .val = pCtx->pOutput};

              int32_t srcType = pCtx->inputType;
              if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) {
                setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
              } else {
                bool exceedMax = false, exceedMin = false;
                taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE, &exceedMax, &exceedMin);
                if (exceedMax || exceedMin) {
                  __compar_fn_t func = getComparFunc((int32_t)pCtx->inputType, 0);
                  if (func(&pCtx->start.val, &pCtx->end.val) <= 0) {
                    COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->start.val : &pCtx->end.val);
                  } else {
                    COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->end.val : &pCtx->start.val);
                  }
                }
              }
#endif
                  break;

                case TSDB_FILL_PREV: {
                  SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
                  colDataAppend(pDst, numOfRows, pkey->pData, false);
                } break;

                case TSDB_FILL_NEXT: {
                } break;

                case TSDB_FILL_NONE:
                default:
                  break;
              }

              pSliceInfo->current +=
                  taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
H
Haojun Liao 已提交
1851 1852
            }
          } else {
H
Haojun Liao 已提交
1853
            // ignore current row, and do nothing
H
Haojun Liao 已提交
1854 1855
          }
        } else {  // it is the last row of current block
H
Haojun Liao 已提交
1856
          doKeepPrevRows(pSliceInfo, pBlock);
H
Haojun Liao 已提交
1857 1858 1859
        }
      }
    }
1860 1861 1862 1863
  }

  // restore the value
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
1864
  if (pResBlock->info.rows == 0) {
1865 1866 1867
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895
  return pResBlock->info.rows == 0 ? NULL : pResBlock;
}

static int32_t initTimesliceInfo(STimeSliceOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfCols) {
  pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
  pInfo->pCols = taosArrayInit(4, sizeof(SColumn));

  if (pInfo->pPrevRow == NULL || pInfo->pCols == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  for (int32_t i = 0; i < numOfCols; ++i) {
    SExprInfo* pExpr = pCtx[i].pExpr;

    SFunctParam* pParam = &pExpr->base.pParam[0];

    SColumn c = *pParam->pCol;
    taosArrayPush(pInfo->pCols, &c);

    SGroupKeys key = {0};
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, c.bytes);
    taosArrayPush(pInfo->pPrevRow, &key);
  }

  return TSDB_CODE_SUCCESS;
1896 1897 1898
}

SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
H
Haojun Liao 已提交
1899
                                           SSDataBlock* pResultBlock, const SNodeListNode* pValNode, SExecTaskInfo* pTaskInfo) {
1900 1901 1902 1903 1904 1905
  STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pOperator == NULL || pInfo == NULL) {
    goto _error;
  }

H
Haojun Liao 已提交
1906 1907 1908 1909 1910
  int32_t code = initTimesliceInfo(pInfo, pInfo->binfo.pCtx, numOfCols);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

1911
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
H
Haojun Liao 已提交
1912
  pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfCols, pValNode);
1913

H
Haojun Liao 已提交
1914
  pInfo->binfo.pRes     = pResultBlock;
H
Haojun Liao 已提交
1915 1916

  pOperator->name       = "TimeSliceOperator";
1917
  //  pOperator->operatorType = OP_AllTimeWindow;
H
Haojun Liao 已提交
1918 1919 1920
  pOperator->blocking   = true;
  pOperator->status     = OP_NOT_OPENED;
  pOperator->pExpr      = pExprInfo;
1921
  pOperator->numOfExprs = numOfCols;
H
Haojun Liao 已提交
1922 1923
  pOperator->info       = pInfo;
  pOperator->pTaskInfo  = pTaskInfo;
1924

H
Haojun Liao 已提交
1925
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyBasicOperatorInfo,
1926 1927
                                         NULL, NULL, NULL);

H
Haojun Liao 已提交
1928
  code = appendDownstream(pOperator, &downstream, 1);
1929 1930
  return pOperator;

L
Liu Jicong 已提交
1931
_error:
1932 1933 1934 1935 1936 1937 1938
  taosMemoryFree(pInfo);
  taosMemoryFree(pOperator);
  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
  return NULL;
}

SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
1939
                                             SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId,
1940
                                             SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo) {
1941 1942 1943 1944 1945 1946
  SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1947 1948 1949 1950 1951 1952 1953 1954
  pInfo->stateCol = *pStateKeyCol;
  pInfo->stateKey.type = pInfo->stateCol.type;
  pInfo->stateKey.bytes = pInfo->stateCol.bytes;
  pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
  if (pInfo->stateKey.pData == NULL) {
    goto _error;
  }

1955 1956 1957 1958
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

  initResultSizeInfo(pOperator, 4096);
  initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
1959
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1960

L
Liu Jicong 已提交
1961
  pInfo->twAggSup = *pTwAggSup;
1962 1963
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

X
Xiaoyu Wang 已提交
1964 1965
  pInfo->tsSlotId = tsSlotId;
  pOperator->name = "StateWindowOperator";
1966
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE;
X
Xiaoyu Wang 已提交
1967 1968 1969
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExpr;
L
Liu Jicong 已提交
1970
  pOperator->numOfExprs = numOfCols;
X
Xiaoyu Wang 已提交
1971 1972
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
1973 1974 1975 1976 1977 1978 1979

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL,
                                         destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  int32_t code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

L
Liu Jicong 已提交
1980
_error:
1981 1982 1983 1984 1985 1986 1987 1988 1989 1990
  pTaskInfo->code = TSDB_CODE_SUCCESS;
  return NULL;
}

void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
}

SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
L
Liu Jicong 已提交
1991 1992
                                            SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId,
                                            STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
1993 1994 1995 1996 1997 1998
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1999 2000
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);
2001 2002 2003 2004 2005 2006 2007 2008

  int32_t code =
      initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->twAggSup = *pTwAggSupp;
2009
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2010 2011
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

L
Liu Jicong 已提交
2012 2013 2014 2015 2016 2017
  pInfo->tsSlotId = tsSlotId;
  pInfo->gap = gap;
  pInfo->binfo.pRes = pResBlock;
  pInfo->winSup.prevTs = INT64_MIN;
  pInfo->reptScan = false;
  pOperator->name = "SessionWindowAggOperator";
2018
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
2019
  pOperator->blocking = true;
L
Liu Jicong 已提交
2020 2021 2022 2023
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->numOfExprs = numOfCols;
  pOperator->info = pInfo;
2024 2025 2026 2027 2028 2029 2030 2031

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL,
                                         destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  pOperator->pTaskInfo = pTaskInfo;

  code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

L
Liu Jicong 已提交
2032
_error:
2033 2034 2035 2036 2037 2038 2039 2040
  if (pInfo != NULL) {
    destroySWindowOperatorInfo(pInfo, numOfCols);
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
2041
}
5
54liuyao 已提交
2042

5
54liuyao 已提交
2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091
bool isFinalInterval(SStreamFinalIntervalOperatorInfo* pInfo) { return pInfo->pChildren != NULL; }

void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
                      SExecTaskInfo* pTaskInfo) {
  for (int32_t k = 0; k < numOfOutput; ++k) {
    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
      continue;
    }
    int32_t code = TSDB_CODE_SUCCESS;
    if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
      code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
      if (code != TSDB_CODE_SUCCESS) {
        qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
        pTaskInfo->code = code;
        longjmp(pTaskInfo->env, code);
      }
    }
  }
}

static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId,
                                  int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
  int32_t size = taosArrayGetSize(pWinArray);
  ASSERT(pInfo->pChildren);
  for (int32_t i = 0; i < size; i++) {
    STimeWindow* pParentWin = taosArrayGet(pWinArray, i);
    SResultRow*  pCurResult = NULL;
    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, pParentWin, true, &pCurResult, 0, pInfo->binfo.pCtx,
                           numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
    for (int32_t j = 0; j < numOfChildren; j++) {
      SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
      SResultRow*               pChResult = NULL;
      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, pParentWin, true, &pChResult, 0, pChInfo->binfo.pCtx,
                             pChildOp->numOfExprs, pChInfo->binfo.rowCellInfoOffset, &pChInfo->aggSup, pTaskInfo);
      compactFunctions(pInfo->binfo.pCtx, pChInfo->binfo.pCtx, numOfOutput, pTaskInfo);
    }
  }
}

bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
  SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId);
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable,
      pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(int64_t)));
  return p1 == NULL;
}

static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
S
shenglian zhou 已提交
2092
                           SArray* pUpdated) {
5
54liuyao 已提交
2093
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
X
Xiaoyu Wang 已提交
2094 2095 2096 2097 2098 2099 2100 2101
  SResultRowInfo*                   pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*                    pTaskInfo = pOperatorInfo->pTaskInfo;
  int32_t                           numOfOutput = pOperatorInfo->numOfExprs;
  int32_t                           step = 1;
  bool                              ascScan = true;
  TSKEY*                            tsCols = NULL;
  SResultRow*                       pResult = NULL;
  int32_t                           forwardRows = 0;
5
54liuyao 已提交
2102 2103 2104 2105

  if (pSDataBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
2106
  } else {
S
shenglian zhou 已提交
2107
    return;
5
54liuyao 已提交
2108
  }
5
54liuyao 已提交
2109

X
Xiaoyu Wang 已提交
2110 2111 2112 2113
  int32_t     startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
  STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
                                            pInfo->interval.precision, NULL);
5
54liuyao 已提交
2114
  while (1) {
5
54liuyao 已提交
2115 2116 2117 2118 2119 2120 2121 2122
    if (isFinalInterval(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) &&
        isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup)) {
      SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
      taosArrayPush(pUpWins, &nextWin);
      rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId,
          pOperatorInfo->numOfExprs, pOperatorInfo->pTaskInfo);
      taosArrayDestroy(pUpWins);
    }
X
Xiaoyu Wang 已提交
2123 2124
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx,
                                          numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2125 2126 2127 2128 2129 2130 2131
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    pos->groupId = tableGroupId;
    pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
    *(int64_t*)pos->key = pResult->win.skey;
S
shenglian zhou 已提交
2132 2133
    forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
                                           TSDB_ORDER_ASC);
5
54liuyao 已提交
2134
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) {
5
54liuyao 已提交
2135
      saveResultRow(pResult, tableGroupId, pUpdated);
5
54liuyao 已提交
2136
    }
5
54liuyao 已提交
2137
    // window start(end) key interpolation
S
shenglian zhou 已提交
2138 2139
    // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos,
    // forwardRows);
5
54liuyao 已提交
2140
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
X
Xiaoyu Wang 已提交
2141 2142
    doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                     tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
2143
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
2144
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
5
54liuyao 已提交
2145 2146 2147 2148 2149 2150 2151
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
    if (startPos < 0) {
      break;
    }
  }
}

5
54liuyao 已提交
2152 2153 2154 2155
static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) {
  taosHashClear(pInfo->aggSup.pResultRowHashTable);
  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
  cleanupResultRowInfo(&pInfo->binfo.resultRowInfo);
2156
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
2157 2158 2159 2160 2161 2162 2163 2164 2165
}

static void clearUpdateDataBlock(SSDataBlock* pBlock) {
  if (pBlock->info.rows <= 0) {
    return;
  }
  blockDataCleanup(pBlock);
}

2166
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
5
54liuyao 已提交
2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177
  ASSERT(pDest->info.capacity >= pSource->info.rows);
  clearUpdateDataBlock(pDest);
  SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
  SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
  // copy timestamp column
  colDataAssign(pDestCol, pSourceCol, pSource->info.rows);
  for (int32_t i = 1; i < pDest->info.numOfCols; i++) {
    SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
    colDataAppendNNULL(pCol, 0, pSource->info.rows);
  }
  pDest->info.rows = pSource->info.rows;
2178 2179
  pDest->info.groupId = pSource->info.groupId;
  pDest->info.type = pSource->info.type;
5
54liuyao 已提交
2180 2181 2182 2183
  blockDataUpdateTsWindow(pDest, 0);
}

static int32_t getChildIndex(SSDataBlock* pBlock) {
2184
  return pBlock->info.childId;
5
54liuyao 已提交
2185 2186
}

5
54liuyao 已提交
2187 2188
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
  SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2189
  SOperatorInfo*                    downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
2190
  SArray*                           pUpdated = taosArrayInit(4, POINTER_BYTES);
5
54liuyao 已提交
2191
  TSKEY                             maxTs = INT64_MIN;
5
54liuyao 已提交
2192 2193 2194 2195 2196

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
2197
    if (pInfo->binfo.pRes->info.rows == 0) {
5
54liuyao 已提交
2198
      pOperator->status = OP_EXEC_DONE;
5
54liuyao 已提交
2199 2200 2201 2202 2203 2204 2205 2206 2207 2208
      if (isFinalInterval(pInfo) || pInfo->pUpdateRes->info.rows == 0) {
        if (!isFinalInterval(pInfo)) {
          // semi interval operator clear disk buffer
          clearStreamIntervalOperator(pInfo);
        }
        return NULL;
      }
      // process the rest of the data
      pOperator->status = OP_OPENED;
      return pInfo->pUpdateRes;
5
54liuyao 已提交
2209
    }
5
54liuyao 已提交
2210
    return pInfo->binfo.pRes;
5
54liuyao 已提交
2211 2212 2213 2214 2215
  }

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
2216
      clearUpdateDataBlock(pInfo->pUpdateRes);
5
54liuyao 已提交
2217 2218
      break;
    }
2219

5
54liuyao 已提交
2220
    if (pBlock->info.type == STREAM_REPROCESS) {
X
Xiaoyu Wang 已提交
2221 2222 2223
      SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
      doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, pInfo->primaryTsIndex, pOperator->numOfExprs,
                     pBlock, pUpWins);
2224
      if (isFinalInterval(pInfo)) {
5
54liuyao 已提交
2225
        int32_t                   childIndex = getChildIndex(pBlock);
X
Xiaoyu Wang 已提交
2226
        SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
2227
        SIntervalAggOperatorInfo* pChildInfo = pChildOp->info;
S
shenglian zhou 已提交
2228 2229 2230 2231
        doClearWindows(&pChildInfo->aggSup, &pChildInfo->binfo, &pChildInfo->interval, pChildInfo->primaryTsIndex,
                       pChildOp->numOfExprs, pBlock, NULL);
        rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->numOfExprs,
                              pOperator->pTaskInfo);
5
54liuyao 已提交
2232 2233
        taosArrayDestroy(pUpWins);
        continue;
2234
      }
5
54liuyao 已提交
2235 2236
      removeResults(pUpWins, pUpdated);
      copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
2237
      taosArrayDestroy(pUpWins);
5
54liuyao 已提交
2238
      break;
2239
    } else if (pBlock->info.type == STREAM_GET_ALL && isFinalInterval(pInfo)) {
5
54liuyao 已提交
2240 2241
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
      continue;
5
54liuyao 已提交
2242
    }
5
54liuyao 已提交
2243

2244
    setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
5
54liuyao 已提交
2245
    doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated);
2246
    if (isFinalInterval(pInfo)) {
S
shenglian zhou 已提交
2247
      int32_t chIndex = getChildIndex(pBlock);
5
54liuyao 已提交
2248 2249 2250 2251 2252 2253 2254 2255 2256
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
        SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
        if (!pChildOp) {
          longjmp(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
        }
        taosArrayPush(pInfo->pChildren, &pChildOp);
      }
S
shenglian zhou 已提交
2257
      SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
2258 2259 2260 2261
      SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
      setInputDataBlock(pChildOp, pChInfo->binfo.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
      doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL);
    }
5
54liuyao 已提交
2262
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
2263
  }
S
shenglian zhou 已提交
2264

5
54liuyao 已提交
2265
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
2266
  if (isFinalInterval(pInfo)) {
5
54liuyao 已提交
2267
    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
5
54liuyao 已提交
2268 2269 2270 2271 2272 2273 2274
  }

  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset);
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
  pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286
  if (pInfo->binfo.pRes->info.rows == 0) {
    pOperator->status = OP_EXEC_DONE;
    if (pInfo->pUpdateRes->info.rows == 0) {
      return NULL;
    }
    // process the rest of the data
    pOperator->status = OP_OPENED;
    return pInfo->pUpdateRes;
  }
  return pInfo->binfo.pRes;
}

S
shenglian zhou 已提交
2287 2288 2289
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                     SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
  SIntervalPhysiNode*               pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
5
54liuyao 已提交
2290
  SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo));
S
shenglian zhou 已提交
2291
  SOperatorInfo*                    pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
2292 2293 2294 2295
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
  pInfo->order = TSDB_ORDER_ASC;
S
shenglian zhou 已提交
2296 2297 2298 2299 2300 2301 2302 2303
  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                .sliding = pIntervalPhyNode->sliding,
                                .intervalUnit = pIntervalPhyNode->intervalUnit,
                                .slidingUnit = pIntervalPhyNode->slidingUnit,
                                .offset = pIntervalPhyNode->offset,
                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pIntervalPhyNode->window.watermark,
5
54liuyao 已提交
2304 2305
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
S
shenglian zhou 已提交
2306
  };
2307
  ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY);
5
54liuyao 已提交
2308 2309 2310
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);
S
shenglian zhou 已提交
2311 2312
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5
54liuyao 已提交
2313 2314 2315
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols,
      pResBlock, keyBufSize, pTaskInfo->id.str);
2316 2317
  ASSERT(numOfCols > 0);
  increaseTs(pInfo->binfo.pCtx);
5
54liuyao 已提交
2318 2319 2320 2321
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
2322
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338
  pInfo->pChildren = NULL;
  if (numOfChild > 0) {
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(SOperatorInfo));
    for (int32_t i = 0; i < numOfChild; i++) {
      SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp) {
        taosArrayPush(pInfo->pChildren, &pChildOp);
        continue;
      }
      goto _error;
    }
  }
  // semi interval operator does not catch result
  if (!isFinalInterval(pInfo)) {
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }
S
shenglian zhou 已提交
2339
  pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
5
54liuyao 已提交
2340 2341
  pInfo->pUpdateRes->info.type = STREAM_REPROCESS;
  blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
2342
  pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
5
54liuyao 已提交
2343 2344 2345 2346 2347 2348 2349 2350 2351 2352

  pOperator->name = "StreamFinalIntervalOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->numOfExprs = numOfCols;
  pOperator->info = pInfo;

S
shenglian zhou 已提交
2353 2354 2355
  pOperator->fpSet =
      createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
  destroyStreamFinalIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
5
54liuyao 已提交
2370
}
5
54liuyao 已提交
2371 2372 2373

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
2374 2375 2376 2377 2378 2379
  void **pIte = NULL;
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
    SArray *pWins = (SArray *) (*pIte);
    taosArrayDestroy(pWins);
  }
  taosHashCleanup(pSup->pResultRows);
5
54liuyao 已提交
2380 2381 2382 2383 2384 2385 2386 2387
  destroyDiskbasedBuf(pSup->pResultBuf);
}

void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
2388 2389 2390
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
2391
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
2392 2393 2394 2395 2396 2397
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
      destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput);
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
5
54liuyao 已提交
2398 2399
}

S
shenglian zhou 已提交
2400
int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock) {
5
54liuyao 已提交
2401 2402 2403
  pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset);
  pBasicInfo->pRes = pResultBlock;
  for (int32_t i = 0; i < numOfCols; ++i) {
2404
    pBasicInfo->pCtx[i].pBuf = NULL;
5
54liuyao 已提交
2405
  }
2406 2407
  ASSERT(numOfCols > 0);
  increaseTs(pBasicInfo->pCtx);
5
54liuyao 已提交
2408 2409 2410 2411 2412 2413 2414 2415
  return TSDB_CODE_SUCCESS;
}

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {
  for (int i = 0; i < nums; i++) {
    pDummy[i].functionId = pCtx[i].functionId;
  }
}
X
Xiaoyu Wang 已提交
2416 2417
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
                    uint8_t type) {
5
54liuyao 已提交
2418 2419
  ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
  SStreamBlockScanInfo* pScanInfo = downstream->info;
X
Xiaoyu Wang 已提交
2420
  pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
5
54liuyao 已提交
2421
  pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
5
54liuyao 已提交
2422 2423
}

2424 2425 2426 2427
int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo));
}

2428 2429 2430 2431 2432 2433 2434
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
  SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
  int32_t numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  int32_t code = TSDB_CODE_OUT_OF_MEMORY;
X
Xiaoyu Wang 已提交
2435
  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
2436
  SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
2437 2438 2439 2440 2441 2442
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

  initResultSizeInfo(pOperator, 4096);

2443
  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
2444 2445 2446
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
2447 2448
  
  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pInfo->binfo.pCtx, numOfCols);
5
54liuyao 已提交
2449 2450 2451
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
X
Xiaoyu Wang 已提交
2452

5
54liuyao 已提交
2453 2454 2455 2456 2457 2458
  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }
  initDummyFunction(pInfo->pDummyCtx, pInfo->binfo.pCtx, numOfCols);

2459 2460 2461
  pInfo->twAggSup = (STimeWindowAggSupp) {.waterMark = pSessionNode->window.watermark,
      .calTrigger = pSessionNode->window.triggerType,
      .maxTs = INT64_MIN};
5
54liuyao 已提交
2462 2463 2464 2465
  initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

  pInfo->primaryTsIndex = tsSlotId;
2466
  pInfo->gap = pSessionNode->gap;
5
54liuyao 已提交
2467 2468 2469 2470 2471 2472 2473
  pInfo->binfo.pRes = pResBlock;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
  pInfo->pDelRes = createOneDataBlock(pResBlock, false);
  blockDataEnsureCapacity(pInfo->pDelRes, 64);
2474
  pInfo->pChildren = NULL;
5
54liuyao 已提交
2475 2476

  pOperator->name = "StreamSessionWindowAggOperator";
2477
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
5
54liuyao 已提交
2478 2479 2480 2481 2482
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->numOfExprs = numOfCols;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
2483 2484 2485
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
2486
  pOperator->pTaskInfo = pTaskInfo;
X
Xiaoyu Wang 已提交
2487
  initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType);
5
54liuyao 已提交
2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502
  code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

_error:
  if (pInfo != NULL) {
    destroyStreamSessionAggOperatorInfo(pInfo, numOfCols);
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

int64_t getSessionWindowEndkey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
2503
  SArray*            pWinInfos = (SArray*)data;
5
54liuyao 已提交
2504 2505 2506 2507 2508 2509
  SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index);
  return pWin->win.ekey;
}
static bool isInWindow(SResultWindowInfo* pWin, TSKEY ts, int64_t gap) {
  int64_t sGap = ts - pWin->win.skey;
  int64_t eGap = pWin->win.ekey - ts;
X
Xiaoyu Wang 已提交
2510
  if ((sGap < 0 && sGap >= -gap) || (eGap < 0 && eGap >= -gap) || (sGap >= 0 && eGap >= 0)) {
5
54liuyao 已提交
2511 2512 2513 2514 2515
    return true;
  }
  return false;
}

X
Xiaoyu Wang 已提交
2516 2517
static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) {
  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
5
54liuyao 已提交
2518 2519 2520 2521
  return taosArrayInsert(pWinInfos, index, &win);
}

static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) {
X
Xiaoyu Wang 已提交
2522
  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
5
54liuyao 已提交
2523 2524 2525
  return taosArrayPush(pWinInfos, &win);
}

2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541
SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) {
  void** ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t));
  SArray* pWinInfos = NULL;
  if (ite == NULL) {
    pWinInfos = taosArrayInit(1024, pAggSup->valueSize);
    taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void *));
  } else {
    pWinInfos = *ite;
  }
  return pWinInfos;
}

SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int64_t gap, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

5
54liuyao 已提交
2542 2543
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
5
54liuyao 已提交
2544
    *pIndex = 0;
5
54liuyao 已提交
2545 2546 2547
    return addNewSessionWindow(pWinInfos, ts);
  }
  // find the first position which is smaller than the key
X
Xiaoyu Wang 已提交
2548
  int32_t            index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isInWindow(pWin, ts, gap)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isInWindow(pWin, ts, gap)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewSessionWindow(pWinInfos, ts);
  }
5
54liuyao 已提交
2570 2571
  *pIndex = index + 1;
  return insertNewSessionWindow(pWinInfos, ts, index + 1);
5
54liuyao 已提交
2572 2573
}

X
Xiaoyu Wang 已提交
2574 2575
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, int32_t start, int64_t gap,
                                SHashObj* pStDeleted) {
5
54liuyao 已提交
2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591
  for (int32_t i = start; i < rows; ++i) {
    if (!isInWindow(pWinInfo, pTs[i], gap)) {
      return i - start;
    }
    if (pWinInfo->win.skey > pTs[i]) {
      if (pStDeleted && pWinInfo->isOutput) {
        taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY));
        pWinInfo->isOutput = false;
      }
      pWinInfo->win.skey = pTs[i];
    }
    pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pTs[i]);
  }
  return rows - start;
}

X
Xiaoyu Wang 已提交
2592
static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,
5
54liuyao 已提交
2593
                                  uint64_t groupId, int32_t numOfOutput, int32_t* rowCellInfoOffset,
X
Xiaoyu Wang 已提交
2594
                                  SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) {
5
54liuyao 已提交
2595 2596
  assert(pWinInfo->win.skey <= pWinInfo->win.ekey);
  // too many time window in query
2597
  int32_t size = taosArrayGetSize(pAggSup->pCurWins);
5
54liuyao 已提交
2598 2599 2600
  if (size > MAX_INTERVAL_TIME_WINDOW) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
  }
X
Xiaoyu Wang 已提交
2601

5
54liuyao 已提交
2602
  if (pWinInfo->pos.pageId == -1) {
2603
    *pResult = getNewResultRow(pAggSup->pResultBuf, groupId, pAggSup->resultRowSize);
5
54liuyao 已提交
2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625
    if (*pResult == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }
    initResultRow(*pResult);

    // add a new result set for a new group
    pWinInfo->pos.pageId = (*pResult)->pageId;
    pWinInfo->pos.offset = (*pResult)->offset;
  } else {
    *pResult = getResultRowByPos(pAggSup->pResultBuf, &pWinInfo->pos);
    if (!(*pResult)) {
      qError("getResultRowByPos return NULL, TID:%s", GET_TASKID(pTaskInfo));
      return TSDB_CODE_FAILED;
    }
  }

  // set time window for current result
  (*pResult)->win = pWinInfo->win;
  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowCellInfoOffset);
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
2626 2627 2628 2629 2630 2631 2632 2633
static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup,
                                  SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock,
                                  SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
                                  int32_t numOutput, SExecTaskInfo* pTaskInfo) {
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId);
  TSKEY*           tsCols = (int64_t*)pColDataInfo->pData;
  int32_t          code = setWindowOutputBuf(pCurWin, pResult, pBinfo->pCtx, pSDataBlock->info.groupId, numOutput,
                                             pBinfo->rowCellInfoOffset, pAggSup, pTaskInfo);
5
54liuyao 已提交
2634 2635 2636
  if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }
5
54liuyao 已提交
2637
  updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, true);
X
Xiaoyu Wang 已提交
2638 2639
  doApplyFunctions(pTaskInfo, pBinfo->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols,
                   pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC);
5
54liuyao 已提交
2640 2641 2642
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
2643 2644 2645 2646 2647
static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                              SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
                              int32_t numOutput, SExecTaskInfo* pTaskInfo) {
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pTaskInfo);
5
54liuyao 已提交
2648 2649
}

X
Xiaoyu Wang 已提交
2650 2651 2652 2653 2654
static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                                   SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex,
                                   int32_t winRows, int32_t numOutput, SExecTaskInfo* pTaskInfo) {
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pTaskInfo);
5
54liuyao 已提交
2655 2656
}

5
54liuyao 已提交
2657 2658
int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) {
  SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex);
X
Xiaoyu Wang 已提交
2659
  int32_t            size = taosArrayGetSize(pWinInfos);
5
54liuyao 已提交
2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i);
    if (!isInWindow(pCurWin, pWinInfo->win.skey, gap)) {
      return i - startIndex - 1;
    }
  }

  return size - startIndex - 1;
}

5
54liuyao 已提交
2671
void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId,
X
Xiaoyu Wang 已提交
2672
                       int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SHashObj* pStUpdated, SHashObj* pStDeleted) {
2673
  SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex);
X
Xiaoyu Wang 已提交
2674 2675 2676
  SResultRow*        pCurResult = NULL;
  setWindowOutputBuf(pCurWin, &pCurResult, pInfo->binfo.pCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset,
                     &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
2677
  num += startIndex + 1;
2678
  ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins));
5
54liuyao 已提交
2679 2680
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < num; i++) {
2681
    SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i);
X
Xiaoyu Wang 已提交
2682 2683 2684
    SResultRow*        pWinResult = NULL;
    setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset,
                       &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
2685 2686 2687 2688 2689 2690 2691
    pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey);
    compactFunctions(pInfo->binfo.pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo);
    taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition));
    if (pWinInfo->isOutput) {
      taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY));
      pWinInfo->isOutput = false;
    }
2692
    taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
5
54liuyao 已提交
2693 2694 2695
  }
}

5
54liuyao 已提交
2696 2697 2698 2699 2700
typedef struct SWinRes {
  TSKEY    ts;
  uint64_t groupId;
} SWinRes;

X
Xiaoyu Wang 已提交
2701 2702 2703
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
                                   SHashObj* pStDeleted) {
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
2704
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2705 2706
  bool                           masterScan = true;
  int32_t                        numOfOutput = pOperator->numOfExprs;
5
54liuyao 已提交
2707
  uint64_t                       groupId = pSDataBlock->info.groupId;
X
Xiaoyu Wang 已提交
2708 2709 2710 2711 2712 2713
  int64_t                        gap = pInfo->gap;
  int64_t                        code = TSDB_CODE_SUCCESS;

  int32_t     step = 1;
  bool        ascScan = true;
  TSKEY*      tsCols = NULL;
5
54liuyao 已提交
2714
  SResultRow* pResult = NULL;
X
Xiaoyu Wang 已提交
2715
  int32_t     winRows = 0;
5
54liuyao 已提交
2716 2717

  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
2718
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
2719 2720
    tsCols = (int64_t*)pColDataInfo->pData;
  } else {
X
Xiaoyu Wang 已提交
2721
    return;
5
54liuyao 已提交
2722
  }
X
Xiaoyu Wang 已提交
2723

5
54liuyao 已提交
2724
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
2725 2726
  for (int32_t i = 0; i < pSDataBlock->info.rows;) {
    int32_t            winIndex = 0;
5
54liuyao 已提交
2727
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], groupId, gap, &winIndex);
X
Xiaoyu Wang 已提交
2728
    winRows = updateSessionWindowInfo(pCurWin, tsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted);
5
54liuyao 已提交
2729 2730 2731 2732 2733
    code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pTaskInfo);
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    // window start(end) key interpolation
X
Xiaoyu Wang 已提交
2734 2735
    // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos,
    // forwardRows,
5
54liuyao 已提交
2736
    //                             pInfo->order, false);
2737
    int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap);
5
54liuyao 已提交
2738 2739 2740
    if (winNum > 0) {
      compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pTaskInfo, pStUpdated, pStDeleted);
    }
5
54liuyao 已提交
2741 2742
    pCurWin->isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
2743 2744
      SWinRes value = {.ts = pCurWin->win.skey, .groupId = groupId};
      code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinRes));
5
54liuyao 已提交
2745 2746 2747 2748
      if (code != TSDB_CODE_SUCCESS) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
      pCurWin->isOutput = true;
5
54liuyao 已提交
2749 2750 2751 2752 2753
    }
    i += winRows;
  }
}

X
Xiaoyu Wang 已提交
2754 2755
static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SOptrBasicInfo* pBinfo, SSDataBlock* pBlock,
                                  int32_t tsIndex, int32_t numOfOutput, int64_t gap, SArray* result) {
5
54liuyao 已提交
2756
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
2757 2758
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
  int32_t          step = 0;
5
54liuyao 已提交
2759
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
2760
    int32_t            winIndex = 0;
2761
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], pBlock->info.groupId, gap, &winIndex);
5
54liuyao 已提交
2762
    step = updateSessionWindowInfo(pCurWin, tsCols, pBlock->info.rows, i, gap, NULL);
2763
    ASSERT(isInWindow(pCurWin, tsCols[i], gap));
5
54liuyao 已提交
2764
    doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pBinfo, numOfOutput);
2765 2766 2767
    if (result) {
      taosArrayPush(result, pCurWin);
    }
5
54liuyao 已提交
2768 2769 2770
  }
}

5
54liuyao 已提交
2771
static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) {
X
Xiaoyu Wang 已提交
2772
  void*  pData = NULL;
5
54liuyao 已提交
2773
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
2774
  while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) {
5
54liuyao 已提交
2775 2776 2777 2778 2779 2780
    void* key = taosHashGetKey(pData, &keyLen);
    ASSERT(keyLen == sizeof(SResultRowPosition));
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    if (pos == NULL) {
      return TSDB_CODE_QRY_OUT_OF_MEMORY;
    }
5
54liuyao 已提交
2781
    pos->groupId = ((SWinRes*)pData)->groupId;
5
54liuyao 已提交
2782
    pos->pos = *(SResultRowPosition*)key;
5
54liuyao 已提交
2783
    *(int64_t*)pos->key = ((SWinRes*)pData)->ts;
5
54liuyao 已提交
2784 2785 2786 2787 2788 2789 2790 2791
    taosArrayPush(pUpdated, &pos);
  }
  return TSDB_CODE_SUCCESS;
}

void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
  blockDataCleanup(pBlock);
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
2792
  while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
5
54liuyao 已提交
2793
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
X
Xiaoyu Wang 已提交
2794
    colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false);
5
54liuyao 已提交
2795 2796
    for (int32_t i = 1; i < pBlock->info.numOfCols; i++) {
      pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
X
Xiaoyu Wang 已提交
2797
      colDataAppendNULL(pColInfoData, pBlock->info.rows);
5
54liuyao 已提交
2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808
    }
    pBlock->info.rows += 1;
    if (pBlock->info.rows + 1 >= pBlock->info.capacity) {
      break;
    }
  }
  if ((*Ite) == NULL) {
    taosHashClear(pStDeleted);
  }
}

X
Xiaoyu Wang 已提交
2809 2810
static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId,
                              int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
2811 2812 2813 2814
  int32_t size = taosArrayGetSize(pWinArray);
  ASSERT(pInfo->pChildren);
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i);
X
Xiaoyu Wang 已提交
2815 2816 2817
    SResultRow*        pCurResult = NULL;
    setWindowOutputBuf(pParentWin, &pCurResult, pInfo->binfo.pCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset,
                       &pInfo->streamAggSup, pTaskInfo);
2818 2819
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
    for (int32_t j = 0; j < numOfChildren; j++) {
X
Xiaoyu Wang 已提交
2820
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
2821
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
2822
      SArray*                        pChWins = getWinInfos(&pChInfo->streamAggSup, groupId);
X
Xiaoyu Wang 已提交
2823 2824
      int32_t                        chWinSize = taosArrayGetSize(pChWins);
      int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey);
2825 2826 2827 2828
      for (int32_t k = index; k > 0 && k < chWinSize; k++) {
        SResultWindowInfo* pcw = taosArrayGet(pChWins, k);
        if (pParentWin->win.skey <= pcw->win.skey && pcw->win.ekey <= pParentWin->win.ekey) {
          SResultRow* pChResult = NULL;
X
Xiaoyu Wang 已提交
2829 2830
          setWindowOutputBuf(pcw, &pChResult, pChInfo->binfo.pCtx, groupId, numOfOutput,
                             pChInfo->binfo.rowCellInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
2831 2832 2833 2834 2835 2836 2837 2838 2839
          compactFunctions(pInfo->binfo.pCtx, pChInfo->binfo.pCtx, numOfOutput, pTaskInfo);
          continue;
        }
        break;
      }
    }
  }
}

X
Xiaoyu Wang 已提交
2840
bool isFinalSession(SStreamSessionAggOperatorInfo* pInfo) { return pInfo->pChildren != NULL; }
2841

X
Xiaoyu Wang 已提交
2842 2843 2844
typedef SResultWindowInfo* (*__get_win_info_)(void*);
SResultWindowInfo* getSessionWinInfo(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getStateWinInfo(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
5
54liuyao 已提交
2845

2846
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed,
5
54liuyao 已提交
2847
    __get_win_info_ fn) {
5
54liuyao 已提交
2848
  // Todo(liuyao) save window to tdb
2849
  void **pIte = NULL;
5
54liuyao 已提交
2850
  size_t keyLen = 0;
2851
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
5
54liuyao 已提交
2852
    uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen);
2853 2854 2855 2856 2857 2858 2859 2860 2861
    SArray *pWins = (SArray *) (*pIte);
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) {
        if (!pSeWin->isClosed) {
          pSeWin->isClosed = true;
          if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
2862
            int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed);
2863 2864
            pSeWin->isOutput = true;
          }
5
54liuyao 已提交
2865
        }
2866
        continue;
5
54liuyao 已提交
2867
      }
2868
      break;
5
54liuyao 已提交
2869 2870 2871 2872 2873
    }
  }
  return TSDB_CODE_SUCCESS;
}

2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885
int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) {
  void **pIte = NULL;
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    SArray *pWins = (SArray *) (*pIte);
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (!pSeWin->isClosed) {
        int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed);
        pSeWin->isOutput = true;
      }
5
54liuyao 已提交
2886 2887 2888 2889 2890
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
2891
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
5
54liuyao 已提交
2892 2893 2894 2895 2896
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2897
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
5
54liuyao 已提交
2898 2899 2900 2901 2902
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
2903
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
2904
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
5
54liuyao 已提交
2905 2906 2907 2908 2909
      doSetOperatorCompleted(pOperator);
    }
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
2910 2911
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
2912
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
2913
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
2914 2915 2916 2917 2918
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
2919

X
Xiaoyu Wang 已提交
2920 2921 2922
    if (pBlock->info.type == STREAM_REPROCESS) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
      doClearSessionWindows(&pInfo->streamAggSup, &pInfo->binfo, pBlock, 0, pOperator->numOfExprs, pInfo->gap, pWins);
2923
      if (isFinalSession(pInfo)) {
X
Xiaoyu Wang 已提交
2924 2925
        int32_t                        childIndex = 0;  // Todo(liuyao) get child id from SSDataBlock
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
2926
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
X
Xiaoyu Wang 已提交
2927 2928
        doClearSessionWindows(&pChildInfo->streamAggSup, &pChildInfo->binfo, pBlock, 0, pChildOp->numOfExprs,
                              pChildInfo->gap, NULL);
5
54liuyao 已提交
2929
        rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->numOfExprs, pOperator->pTaskInfo);
2930 2931
      }
      taosArrayDestroy(pWins);
5
54liuyao 已提交
2932
      continue;
2933
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
2934 2935
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getSessionWinInfo);
      continue;
5
54liuyao 已提交
2936
    }
5
54liuyao 已提交
2937

2938 2939
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
2940
    if (isFinalSession(pInfo)) {
X
Xiaoyu Wang 已提交
2941
      int32_t         childIndex = 0;  // Todo(liuyao) get child id from SSDataBlock
2942
      SOptrBasicInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
5
54liuyao 已提交
2943
      doStreamSessionAggImpl(pOperator, pBlock, NULL, NULL);
2944
    }
5
54liuyao 已提交
2945
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted);
5
54liuyao 已提交
2946
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
2947 2948 2949
  }
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
2950

5
54liuyao 已提交
2951
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
X
Xiaoyu Wang 已提交
2952
                     getSessionWinInfo);
5
54liuyao 已提交
2953
  copyUpdateResult(pStUpdated, pUpdated);
5
54liuyao 已提交
2954
  taosHashCleanup(pStUpdated);
5
54liuyao 已提交
2955

5
54liuyao 已提交
2956
  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
X
Xiaoyu Wang 已提交
2957
                        pInfo->binfo.rowCellInfoOffset);
5
54liuyao 已提交
2958 2959 2960 2961 2962 2963
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
2964
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
2965 2966
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}
2967

2968 2969 2970 2971
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream,
    SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
  int32_t        code = TSDB_CODE_OUT_OF_MEMORY;
  SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo);
2972 2973 2974 2975
  if (pOperator == NULL) {
    goto _error;
  }
  pOperator->name = "StreamFinalSessionWindowAggOperator";
2976
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION;
2977
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2978
  pInfo->pChildren = taosArrayInit(8, sizeof(void*));
2979
  for (int32_t i = 0; i < numOfChild; i++) {
X
Xiaoyu Wang 已提交
2980
    SOperatorInfo* pChild =
2981
        createStreamSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo);
2982 2983 2984 2985 2986 2987 2988 2989 2990
    if (pChild == NULL) {
      goto _error;
    }
    taosArrayPush(pInfo->pChildren, &pChild);
  }
  return pOperator;

_error:
  if (pInfo != NULL) {
2991
    destroyStreamSessionAggOperatorInfo(pInfo, pOperator->numOfExprs);
2992 2993 2994 2995 2996 2997 2998
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5
54liuyao 已提交
2999 3000

void destroyStreamStateOperatorInfo(void* param, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
3001
  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;
5
54liuyao 已提交
3002 3003 3004 3005 3006 3007
  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
3008
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
5
54liuyao 已提交
3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
      destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput);
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
}

int64_t getStateWinTsKey(void* data, int32_t index) {
  SStateWindowInfo* pStateWin = taosArrayGet(data, index);
  return pStateWin->winInfo.win.ekey;
}

X
Xiaoyu Wang 已提交
3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033
SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
3034 3035 3036 3037 3038 3039 3040 3041
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayPush(pWinInfos, &win);
}

X
Xiaoyu Wang 已提交
3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053
SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayInsert(pWinInfos, index, &win);
}

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {
  if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) {
    return true;
  }
  return false;
}

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {
  return pKeyData && compareVal(pKeyData, &pWin->stateKey);
}

3073 3074 3075
SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
X
Xiaoyu Wang 已提交
3076 3077
  int32_t           size = taosArrayGetSize(pWinInfos);
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index + 1;
      return pWin;
    }
  }
  *pIndex = 0;
  return NULL;
}

3098 3099 3100 3101
SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts,
    uint64_t groupId, char* pKeyData, SColumn* pCol, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
5
54liuyao 已提交
3102 3103 3104 3105 3106
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    *pIndex = 0;
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
X
Xiaoyu Wang 已提交
3107
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
  *pIndex = index + 1;
  return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol);
}

X
Xiaoyu Wang 已提交
3141 3142
int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, SColumnInfoData* pKeyCol, int32_t rows,
                              int32_t start, bool* allEqual, SHashObj* pSeDelete) {
5
54liuyao 已提交
3143 3144 3145 3146 3147
  *allEqual = true;
  SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex);
  for (int32_t i = start; i < rows; ++i) {
    char* pKeyData = colDataGetData(pKeyCol, i);
    if (!isTsInWindow(pWinInfo, pTs[i])) {
X
Xiaoyu Wang 已提交
3148
      if (isEqualStateKey(pWinInfo, pKeyData)) {
5
54liuyao 已提交
3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162
        int32_t size = taosArrayGetSize(pWinInfos);
        if (winIndex + 1 < size) {
          SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1);
          // ts belongs to the next window
          if (pTs[i] >= pNextWin->winInfo.win.skey) {
            return i - start;
          }
        }
      } else {
        return i - start;
      }
    }
    if (pWinInfo->winInfo.win.skey > pTs[i]) {
      if (pSeDelete && pWinInfo->winInfo.isOutput) {
X
Xiaoyu Wang 已提交
3163 3164
        taosHashPut(pSeDelete, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &pWinInfo->winInfo.win.skey,
                    sizeof(TSKEY));
5
54liuyao 已提交
3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177
        pWinInfo->winInfo.isOutput = false;
      }
      pWinInfo->winInfo.win.skey = pTs[i];
    }
    pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]);
    if (!isEqualStateKey(pWinInfo, pKeyData)) {
      *allEqual = false;
    }
  }
  return rows - start;
}

void deleteWindow(SArray* pWinInfos, int32_t index) {
X
Xiaoyu Wang 已提交
3178
  ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos));
5
54liuyao 已提交
3179 3180 3181
  taosArrayRemove(pWinInfos, index);
}

X
Xiaoyu Wang 已提交
3182 3183
static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int32_t tsIndex, SColumn* pCol,
                                int32_t keyIndex, SHashObj* pSeUpdated, SHashObj* pSeDeleted) {
5
54liuyao 已提交
3184 3185
  SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
  SColumnInfoData* pKeyColInfo = taosArrayGet(pBlock->pDataBlock, keyIndex);
X
Xiaoyu Wang 已提交
3186 3187 3188
  TSKEY*           tsCol = (TSKEY*)pTsColInfo->pData;
  bool             allEqual = false;
  int32_t          step = 1;
5
54liuyao 已提交
3189
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
3190 3191
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
3192
    SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], pBlock->info.groupId, &winIndex);
5
54liuyao 已提交
3193 3194 3195
    if (!pCurWin) {
      continue;
    }
3196
    step = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual,
X
Xiaoyu Wang 已提交
3197
                                 pSeDeleted);
5
54liuyao 已提交
3198 3199 3200
    ASSERT(isTsInWindow(pCurWin, tsCol[i]) || isEqualStateKey(pCurWin, pKeyData));
    taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win);
    taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
3201
    deleteWindow(pAggSup->pCurWins, winIndex);
5
54liuyao 已提交
3202 3203 3204
  }
}

X
Xiaoyu Wang 已提交
3205 3206 3207
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated,
                                 SHashObj* pStDeleted) {
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3208
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3209 3210 3211 3212 3213 3214 3215 3216 3217
  bool                         masterScan = true;
  int32_t                      numOfOutput = pOperator->numOfExprs;
  int64_t                      groupId = pSDataBlock->info.groupId;
  int64_t                      code = TSDB_CODE_SUCCESS;
  int32_t                      step = 1;
  bool                         ascScan = true;
  TSKEY*                       tsCols = NULL;
  SResultRow*                  pResult = NULL;
  int32_t                      winRows = 0;
5
54liuyao 已提交
3218
  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
3219 3220
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
3221
  } else {
X
Xiaoyu Wang 已提交
3222
    return;
5
54liuyao 已提交
3223
  }
X
Xiaoyu Wang 已提交
3224

5
54liuyao 已提交
3225
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
3226 3227 3228 3229 3230
  SColumnInfoData*     pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
  for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
    bool              allEqual = true;
3231 3232 3233 3234 3235
    SStateWindowInfo* pCurWin = 
        getStateWindow(pAggSup, tsCols[i], pSDataBlock->info.groupId, pKeyData,
            &pInfo->stateCol, &winIndex);
    winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, pKeyColInfo,
        pSDataBlock->info.rows, i, &allEqual, pInfo->pSeDeleted);
5
54liuyao 已提交
3236 3237 3238
    if (!allEqual) {
      taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win);
      taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
3239
      deleteWindow(pAggSup->pCurWins, winIndex);
5
54liuyao 已提交
3240 3241
      continue;
    }
X
Xiaoyu Wang 已提交
3242
    code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pTaskInfo);
5
54liuyao 已提交
3243 3244 3245 3246 3247
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    pCurWin->winInfo.isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
3248 3249 3250
      SWinRes value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId};
      code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition),
          &value, sizeof(SWinRes));
5
54liuyao 已提交
3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264
      if (code != TSDB_CODE_SUCCESS) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
      pCurWin->winInfo.isOutput = true;
    }
  }
}

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3265
  SOptrBasicInfo*              pBInfo = &pInfo->binfo;
5
54liuyao 已提交
3266 3267 3268 3269 3270
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
3271
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
3272
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
5
54liuyao 已提交
3273 3274 3275 3276 3277
      doSetOperatorCompleted(pOperator);
    }
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
3278 3279
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
3280
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
3281
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
3282 3283 3284 3285 3286
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
3287

5
54liuyao 已提交
3288
    if (pBlock->info.type == STREAM_REPROCESS) {
X
Xiaoyu Wang 已提交
3289 3290
      doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
                          pSeUpdated, pInfo->pSeDeleted);
5
54liuyao 已提交
3291
      continue;
3292
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
3293 3294
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getStateWinInfo);
      continue;
5
54liuyao 已提交
3295
    }
3296 3297 3298

    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
3299 3300 3301 3302 3303
    doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
X
Xiaoyu Wang 已提交
3304

5
54liuyao 已提交
3305
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
X
Xiaoyu Wang 已提交
3306
                     getStateWinInfo);
5
54liuyao 已提交
3307
  copyUpdateResult(pSeUpdated, pUpdated);
5
54liuyao 已提交
3308 3309 3310
  taosHashCleanup(pSeUpdated);

  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
X
Xiaoyu Wang 已提交
3311
                        pInfo->binfo.rowCellInfoOffset);
5
54liuyao 已提交
3312 3313 3314 3315 3316 3317
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
3318
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
3319 3320 3321
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

3322 3323 3324 3325
int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo));
}

X
Xiaoyu Wang 已提交
3326 3327 3328 3329 3330 3331
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
  SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
3332
  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
3333

X
Xiaoyu Wang 已提交
3334 3335
  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3336 3337 3338 3339
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

X
Xiaoyu Wang 已提交
3340
  int32_t    numOfCols = 0;
5
54liuyao 已提交
3341 3342 3343 3344
  SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
  initResultSizeInfo(pOperator, 4096);
3345
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
X
Xiaoyu Wang 已提交
3346 3347
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
5
54liuyao 已提交
3348 3349
      .calTrigger = pStateNode->window.triggerType,
      .maxTs = INT64_MIN,
X
Xiaoyu Wang 已提交
3350
  };
5
54liuyao 已提交
3351
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
3352 3353

  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
3354 3355 3356 3357
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

3358
  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pInfo->binfo.pCtx, numOfCols);
5
54liuyao 已提交
3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }

  initDummyFunction(pInfo->pDummyCtx, pInfo->binfo.pCtx, numOfCols);
  pInfo->primaryTsIndex = tsSlotId;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
  pInfo->pDelRes = createOneDataBlock(pResBlock, false);
  blockDataEnsureCapacity(pInfo->pDelRes, 64);
  pInfo->pChildren = NULL;

  pOperator->name = "StreamStateAggOperator";
3379
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
5
54liuyao 已提交
3380 3381 3382 3383 3384 3385
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
  pOperator->numOfExprs = numOfCols;
  pOperator->pExpr = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
3386 3387 3388
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL,
                                         destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType);
5
54liuyao 已提交
3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
  return pOperator;

_error:
  destroyStreamStateOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
3402 3403 3404 3405

typedef struct SMergeIntervalAggOperatorInfo {
  SIntervalAggOperatorInfo intervalAggOperatorInfo;

S
shenglian zhou 已提交
3406 3407 3408
  bool         hasGroupId;
  uint64_t     groupId;
  SSDataBlock* prefetchedBlock;
3409
  bool         inputBlocksFinished;
3410 3411 3412
} SMergeIntervalAggOperatorInfo;

void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) {
3413 3414 3415 3416
  SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
  destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput);
}

3417
static int32_t outputMergeIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock, TSKEY wstartTs) {
S
shenglian zhou 已提交
3418 3419 3420 3421
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
  bool                           ascScan = (iaInfo->order == TSDB_ORDER_ASC);
3422

3423 3424 3425 3426
  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId);
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf,
                                                            GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
  ASSERT(p1 != NULL);
3427

3428 3429 3430 3431
  finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, iaInfo->binfo.pCtx, pOperatorInfo->pExpr,
                                       pOperatorInfo->numOfExprs, iaInfo->binfo.rowCellInfoOffset, pResultBlock,
                                       pTaskInfo);
  taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
3432

3433
  return 0;
3434 3435 3436
}

static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
S
shenglian zhou 已提交
3437 3438 3439
                                   int32_t scanFlag, SSDataBlock* pResultBlock) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
3440 3441 3442 3443 3444

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

  int32_t     startPos = 0;
  int32_t     numOfOutput = pOperatorInfo->numOfExprs;
3445
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
3446 3447 3448 3449
  uint64_t    tableGroupId = pBlock->info.groupId;
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

3450 3451 3452
  STimeWindow win;
  win.skey = blockStartTs;
  win.ekey = taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1;
3453

3454
  //TODO: remove the hash table usage (groupid + winkey => result row position)
3455
  int32_t ret =
3456 3457
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, iaInfo->binfo.pCtx,
                             numOfOutput, iaInfo->binfo.rowCellInfoOffset, &iaInfo->aggSup, pTaskInfo);
3458 3459 3460 3461
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
  }

3462 3463 3464
  TSKEY currTs = blockStartTs;
  TSKEY currPos = startPos;
  STimeWindow currWin = win;
3465
  while (1) {
3466 3467
    ++currPos;
    if (currPos >= pBlock->info.rows) {
3468 3469
      break;
    }
3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487
    if (tsCols[currPos] == currTs) {
      continue;
    } else {
      updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
      doApplyFunctions(pTaskInfo, iaInfo->binfo.pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos,
                       currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->order);

      outputMergeIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs);

      currTs = tsCols[currPos];
      currWin.skey = currTs;
      currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1;
      startPos = currPos;
      ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, iaInfo->binfo.pCtx,
                                 numOfOutput, iaInfo->binfo.rowCellInfoOffset, &iaInfo->aggSup, pTaskInfo);
      if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
3488 3489
    }
  }
3490 3491 3492
  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
  doApplyFunctions(pTaskInfo, iaInfo->binfo.pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos,
                   currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->order);
3493

3494
  outputMergeIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs);
3495 3496 3497
}

static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
S
shenglian zhou 已提交
3498
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
3499 3500

  SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info;
S
shenglian zhou 已提交
3501
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
3502 3503 3504 3505
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

3506
  SSDataBlock* pRes = iaInfo->binfo.pRes;
3507
  blockDataCleanup(pRes);
3508
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);
3509

3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520
  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
        miaInfo->groupId = pBlock->info.groupId;
      }
3521

3522 3523 3524 3525
      if (pBlock == NULL) {
        miaInfo->inputBlocksFinished = true;
        break;
      }
3526

3527 3528 3529 3530 3531 3532 3533
      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
        miaInfo->groupId = pBlock->info.groupId;
      } else if (miaInfo->groupId != pBlock->info.groupId) {
        miaInfo->prefetchedBlock = pBlock;
        break;
      }
3534

3535 3536 3537
      getTableScanInfo(pOperator, &iaInfo->order, &scanFlag);
      setInputDataBlock(pOperator, iaInfo->binfo.pCtx, pBlock, iaInfo->order, scanFlag, true);
      doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);
3538

3539 3540 3541 3542 3543 3544
      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
        break;
      }
    }

    pRes->info.groupId = miaInfo->groupId;
3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556
  }

  if (pRes->info.rows == 0) {
    doSetOperatorCompleted(pOperator);
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
S
shenglian zhou 已提交
3557 3558
                                               SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
                                               SExecTaskInfo* pTaskInfo) {
3559
  SMergeIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
S
shenglian zhou 已提交
3560
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
3561 3562 3563 3564
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

S
shenglian zhou 已提交
3565
  SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
3566

3567 3568 3569 3570 3571 3572 3573
  iaInfo->win = pTaskInfo->window;
  iaInfo->order = TSDB_ORDER_ASC;
  iaInfo->interval = *pInterval;

  iaInfo->execModel = pTaskInfo->execModel;

  iaInfo->primaryTsIndex = primaryTsSlotId;
3574 3575 3576 3577 3578

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);

  int32_t code =
3579
      initAggInfo(&iaInfo->binfo, &iaInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
3580

3581
  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
3582

3583 3584 3585
  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(iaInfo->binfo.pCtx, numOfCols, iaInfo);
  if (iaInfo->timeWindowInterpo) {
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
3586 3587
  }

3588 3589
  //  iaInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo);
  if (code != TSDB_CODE_SUCCESS /* || iaInfo->pTableQueryInfo == NULL*/) {
3590 3591 3592
    goto _error;
  }

3593
  initResultRowInfo(&iaInfo->binfo.resultRowInfo);
3594 3595 3596 3597 3598 3599 3600 3601

  pOperator->name = "TimeMergeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL;
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
  pOperator->pExpr = pExprInfo;
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->numOfExprs = numOfCols;
3602
  pOperator->info = miaInfo;
3603

3604
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL,
3605
                                         destroyMergeIntervalOperatorInfo, NULL, NULL, NULL);
3606 3607 3608 3609 3610 3611 3612 3613 3614

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
3615 3616
  destroyMergeIntervalOperatorInfo(miaInfo, numOfCols);
  taosMemoryFreeClear(miaInfo);
3617 3618 3619 3620
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}