timewindowoperator.c 139.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
15
#include "executorimpl.h"
X
Xiaoyu Wang 已提交
16
#include "function.h"
5
54liuyao 已提交
17
#include "functionMgt.h"
L
Liu Jicong 已提交
18 19
#include "tdatablock.h"
#include "ttime.h"
H
Haojun Liao 已提交
20
#include "tfill.h"
21 22 23 24 25 26

typedef enum SResultTsInterpType {
  RESULT_ROW_START_INTERP = 1,
  RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType;

5
54liuyao 已提交
27
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator);
5
54liuyao 已提交
28

29 30 31 32 33
static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo);

static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult);
static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult);

H
Haojun Liao 已提交
34 35 36 37 38 39 40 41 42 43 44 45
///*
// * There are two cases to handle:
// *
// * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including
// * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey.
// * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be
// *    merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there
// *    is a previous result generated or not.
// */
//static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) {
//  // do nothing
//}
46

X
Xiaoyu Wang 已提交
47
static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69

static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w,
                                      bool ascQuery) {
  if (ascQuery) {
    getAlignQueryTimeWindow(pInterval, precision, ts, w);
  } else {
    // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
    getAlignQueryTimeWindow(pInterval, precision, ts, w);

    int64_t key = w->skey;
    while (key < ts) {  // moving towards end
      key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision);
      if (key >= ts) {
        break;
      }

      w->skey = key;
    }
  }
}

// get the correct time window according to the handled timestamp
X
Xiaoyu Wang 已提交
70 71
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
                                int32_t precision, STimeWindow* win) {
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
  STimeWindow w = {0};

  if (pResultRowInfo->cur.pageId == -1) {  // the first window, from the previous stored value
    getInitialStartTimeWindow(pInterval, precision, ts, &w, true);
    w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
  } else {
    w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win;
  }

  if (w.skey > ts || w.ekey < ts) {
    if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
      w.skey = taosTimeTruncate(ts, pInterval, precision);
      w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
    } else {
      int64_t st = w.skey;

      if (st > ts) {
        st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
      }

      int64_t et = st + pInterval->interval - 1;
      if (et < ts) {
        st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
      }

      w.skey = st;
      w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
    }
  }
  return w;
}

static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
106
                                      int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
107 108 109 110 111 112 113 114 115 116 117 118
                                      SExecTaskInfo* pTaskInfo) {
  assert(win->skey <= win->ekey);
  SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                  masterscan, tableGroupId, pTaskInfo, true, pAggSup);

  if (pResultRow == NULL) {
    *pResult = NULL;
    return TSDB_CODE_SUCCESS;
  }

  // set time window for current result
  pResultRow->win = (*win);
119

120
  *pResult = pResultRow;
121
  setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
122

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
  return TSDB_CODE_SUCCESS;
}

static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) {
  int64_t* ts = (int64_t*)pColData->pData;
  int32_t  delta = includeEndpoint ? 1 : 0;

  int64_t duration = pWin->ekey - pWin->skey + delta;
  ts[2] = duration;            // set the duration
  ts[3] = pWin->skey;          // window start key
  ts[4] = pWin->ekey + delta;  // window end key
}

static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) {
  pRowSup->win.ekey = ts;
  pRowSup->prevTs = ts;
  pRowSup->numOfRows += 1;
}

static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex) {
  pRowSup->startRowIndex = rowIndex;
  pRowSup->numOfRows = 0;
  pRowSup->win.skey = tsList[rowIndex];
}

static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
                                                   int16_t pos, int16_t order, int64_t* pData) {
150
  int32_t forwardRows = 0;
151 152 153 154

  if (order == TSDB_ORDER_ASC) {
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
    if (end >= 0) {
155
      forwardRows = end;
156 157

      if (pData[end + pos] == ekey) {
158
        forwardRows += 1;
159 160 161
      }
    }
  } else {
162
    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
163
    if (end >= 0) {
164
      forwardRows = end;
165

166
      if (pData[end + pos] == ekey) {
167
        forwardRows += 1;
168 169
      }
    }
X
Xiaoyu Wang 已提交
170 171 172 173 174 175 176 177
    //    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
    //    if (end >= 0) {
    //      forwardRows = pos - end;
    //
    //      if (pData[end] == ekey) {
    //        forwardRows += 1;
    //      }
    //    }
178 179
  }

180 181
  assert(forwardRows >= 0);
  return forwardRows;
182 183
}

5
54liuyao 已提交
184
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
  int32_t midPos = -1;
  int32_t numOfRows;

  if (num <= 0) {
    return -1;
  }

  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);

  TSKEY*  keyList = (TSKEY*)pValue;
  int32_t firstPos = 0;
  int32_t lastPos = num - 1;

  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller than the key
    while (1) {
201 202 203 204 205 206 207 208 209 210 211
      if (key >= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key < keyList[lastPos]) {
        lastPos += 1;
        if (lastPos >= num) {
          return -1;
        } else {
          return lastPos;
        }
      }
212 213 214 215 216 217

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < keyList[midPos]) {
        firstPos = midPos + 1;
218 219
      } else if (key > keyList[midPos]) {
        lastPos = midPos - 1;
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger than the key
    while (1) {
      if (key <= keyList[firstPos]) return firstPos;
      if (key == keyList[lastPos]) return lastPos;

      if (key > keyList[lastPos]) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1u) + firstPos;

      if (key < keyList[midPos]) {
        lastPos = midPos - 1;
      } else if (key > keyList[midPos]) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
  }

  return midPos;
}

X
Xiaoyu Wang 已提交
255 256
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
                                 __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) {
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
  assert(startPos >= 0 && startPos < pDataBlockInfo->rows);

  int32_t num = -1;
  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);

  if (order == TSDB_ORDER_ASC) {
    if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
      }
    } else {
      num = pDataBlockInfo->rows - startPos;
      if (item != NULL) {
        item->lastKey = pDataBlockInfo->window.ekey + step;
      }
    }
  } else {  // desc
    if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
      num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
      if (item != NULL) {
278
        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
279 280
      }
    } else {
281
      num = pDataBlockInfo->rows - startPos;
282
      if (item != NULL) {
283
        item->lastKey = pDataBlockInfo->window.ekey + step;
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
      }
    }
  }

  assert(num >= 0);
  return num;
}

static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
    tw->skey += pInterval->sliding * factor;
    tw->ekey = tw->skey + pInterval->interval - 1;
    return;
  }

  int64_t key = tw->skey, interval = pInterval->interval;
  // convert key to second
  key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;

  if (pInterval->intervalUnit == 'y') {
    interval *= 12;
  }

  struct tm tm;
  time_t    t = (time_t)key;
  taosLocalTime(&t, &tm);

  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision);

  mon = (int)(mon + interval);
  tm.tm_year = mon / 12;
  tm.tm_mon = mon % 12;
  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision);

  tw->ekey -= 1;
}

325 326 327
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex,
                               TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
  SqlFunctionCtx* pCtx = pSup->pCtx;
328

329
  int32_t index = 1;
330
  for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
H
Haojun Liao 已提交
331
    if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
332 333 334 335
      pCtx[k].start.key = INT64_MIN;
      continue;
    }

X
Xiaoyu Wang 已提交
336
    SFunctParam*     pParam = &pCtx[k].param[0];
337 338
    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId);

339
    ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey);
340

341
    double v1 = 0, v2 = 0, v = 0;
342
    if (prevRowIndex == -1) {
343
      SGroupKeys* p = taosArrayGet(pPrevValues, index);
344
      GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData);
345
    } else {
346
      GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex));
347 348
    }

349
    GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
350

351
#if 0
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
    if (functionId == FUNCTION_INTERP) {
      if (type == RESULT_ROW_START_INTERP) {
        pCtx[k].start.key = prevTs;
        pCtx[k].start.val = v1;

        pCtx[k].end.key = curTs;
        pCtx[k].end.val = v2;

        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
          if (prevRowIndex == -1) {
            //            pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];
          } else {
            pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes;
          }

          pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes;
        }
      }
    } else if (functionId == FUNCTION_TWA) {
371 372
#endif

X
Xiaoyu Wang 已提交
373 374 375
    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
    SPoint point = (SPoint){.key = windowKey, .val = &v};
376

X
Xiaoyu Wang 已提交
377
    taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
378

X
Xiaoyu Wang 已提交
379 380 381 382 383 384
    if (type == RESULT_ROW_START_INTERP) {
      pCtx[k].start.key = point.key;
      pCtx[k].start.val = v;
    } else {
      pCtx[k].end.key = point.key;
      pCtx[k].end.val = v;
385
    }
X
Xiaoyu Wang 已提交
386 387 388

    index += 1;
  }
389
#if 0
390
  }
391
#endif
392 393 394 395 396 397 398 399 400 401 402 403 404 405
}

static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
  if (type == RESULT_ROW_START_INTERP) {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].start.key = INT64_MIN;
    }
  } else {
    for (int32_t k = 0; k < numOfOutput; ++k) {
      pCtx[k].end.key = INT64_MIN;
    }
  }
}

406 407
static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock, const TSKEY* tsCols,
    STimeWindow* win, SExprSupp* pSup) {
X
Xiaoyu Wang 已提交
408
  bool ascQuery = (pInfo->order == TSDB_ORDER_ASC);
409

410
  TSKEY curTs = tsCols[pos];
411 412

  SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
X
Xiaoyu Wang 已提交
413
  TSKEY       lastTs = *(int64_t*)pTsKey->pData;
414 415 416 417 418

  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
  // start exactly from this point, no need to do interpolation
  TSKEY key = ascQuery ? win->skey : win->ekey;
  if (key == curTs) {
419
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
420 421 422
    return true;
  }

423 424
  // it is the first time window, no need to do interpolation
  if (pTsKey->isNull && pos == 0) {
425
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
426 427
  } else {
    TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]);
428 429
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key,
                              RESULT_ROW_START_INTERP, pSup);
430 431 432 433 434
  }

  return true;
}

435
static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup,
X
Xiaoyu Wang 已提交
436 437
                                            int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols,
                                            TSKEY blockEkey, STimeWindow* win) {
438
  int32_t order = pInfo->order;
439 440

  TSKEY actualEndKey = tsCols[endRowIndex];
441
  TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey;
442 443

  // not ended in current data block, do not invoke interpolation
444
  if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) {
445
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
446 447 448
    return false;
  }

449
  // there is actual end point of current time window, no interpolation needs
450
  if (key == actualEndKey) {
451
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
452 453 454
    return true;
  }

455
  int32_t nextRowIndex = endRowIndex + 1;
456 457 458
  assert(nextRowIndex >= 0);

  TSKEY nextKey = tsCols[nextRowIndex];
459 460
  doTimeWindowInterpolation(pInfo->pPrevValues, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key,
                            RESULT_ROW_END_INTERP, pSup);
461 462 463 464
  return true;
}

static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
5
54liuyao 已提交
465
                                      TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
X
Xiaoyu Wang 已提交
466
  bool ascQuery = (order == TSDB_ORDER_ASC);
467 468 469 470 471 472 473 474 475 476

  int32_t precision = pInterval->precision;
  getNextTimeWindow(pInterval, precision, order, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
      (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) {
    return -1;
  }

477
  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
478 479 480 481
  int32_t startPos = 0;

  // tumbling time window query, a special case of sliding time window query
  if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
482
    startPos = prevPosition + 1;
483
  } else {
484
    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
485 486
      startPos = 0;
    } else {
487
      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
    }
  }

  /* interp query with fill should not skip time window */
  //  if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) {
  //    return startPos;
  //  }

  /*
   * This time window does not cover any data, try next time window,
   * this case may happen when the time window is too small
   */
  if (primaryKeys == NULL) {
    if (ascQuery) {
      assert(pDataBlockInfo->window.skey <= pNext->ekey);
    } else {
      assert(pDataBlockInfo->window.ekey >= pNext->skey);
    }
  } else {
    if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->skey = pNext->ekey - pInterval->interval + 1;
      }
    } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
      TSKEY next = primaryKeys[startPos];
      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
        pNext->skey = taosTimeTruncate(next, pInterval, precision);
        pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
      } else {
        pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
        pNext->ekey = pNext->skey + pInterval->interval - 1;
      }
    }
  }

  return startPos;
}

531 532
static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
  ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548
  if (type == RESULT_ROW_START_INTERP) {
    return pResult->startInterp == true;
  } else {
    return pResult->endInterp == true;
  }
}

static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
  assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
  if (type == RESULT_ROW_START_INTERP) {
    pResult->startInterp = true;
  } else {
    pResult->endInterp = true;
  }
}

549 550
static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult, STimeWindow* win, int32_t startPos,
                                        int32_t forwardRows, SExprSupp* pSup) {
551
  if (!pInfo->timeWindowInterpo) {
552 553 554
    return;
  }

555
  ASSERT(pBlock != NULL);
556 557 558 559 560
  if (pBlock->pDataBlock == NULL) {
    //    tscError("pBlock->pDataBlock == NULL");
    return;
  }

561
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
562 563

  TSKEY* tsCols = (TSKEY*)(pColInfo->pData);
564
  bool   done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
565
  if (!done) {  // it is not interpolated, now start to generated the interpolated value
566
    bool interp = setTimeWindowInterpolationStartTs(pInfo, startPos, pBlock, tsCols, win, pSup);
567 568 569 570
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
    }
  } else {
571
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_START_INTERP);
572 573 574 575 576 577 578 579
  }

  // point interpolation does not require the end key time window interpolation.
  //  if (pointInterpQuery) {
  //    return;
  //  }

  // interpolation query does not generate the time window end interpolation
580
  done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
581
  if (!done) {
582
    int32_t endRowIndex = startPos + forwardRows - 1;
583

584
    TSKEY endKey = (pInfo->order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey;
585
    bool  interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win);
586 587 588 589
    if (interp) {
      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
    }
  } else {
590
    setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP);
591 592 593
  }
}

594 595
static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) {
  if (pBlock->pDataBlock == NULL) {
596 597 598
    return;
  }

599 600 601 602 603 604 605
  size_t num = taosArrayGetSize(pPrevKeys);
  for (int32_t k = 0; k < num; ++k) {
    SColumn* pc = taosArrayGet(pCols, k);

    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId);

    SGroupKeys* pkey = taosArrayGet(pPrevKeys, k);
X
Xiaoyu Wang 已提交
606
    for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) {
607 608 609 610 611 612 613 614 615 616 617 618 619 620
      if (colDataIsNull_s(pColInfo, i)) {
        continue;
      }

      char* val = colDataGetData(pColInfo, i);
      if (IS_VAR_DATA_TYPE(pkey->type)) {
        memcpy(pkey->pData, val, varDataTLen(val));
        ASSERT(varDataTLen(val) <= pkey->bytes);
      } else {
        memcpy(pkey->pData, val, pkey->bytes);
      }

      break;
    }
621 622 623
  }
}

624 625 626 627
static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo,
                                       SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) {
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;

628
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
629
  SExprSupp* pSup = &pOperatorInfo->exprSupp;
630

631
  int32_t  startPos = 0;
632
  int32_t  numOfOutput = pSup->numOfExprs;
633
  uint64_t groupId = pBlock->info.groupId;
634

635
  SResultRow* pResult = NULL;
636

637 638
  while (1) {
    SListNode* pn = tdListGetHead(pResultRowInfo->openWindow);
639

640 641 642 643
    SResultRowPosition* p1 = (SResultRowPosition*)pn->data;
    if (p->pageId == p1->pageId && p->offset == p1->offset) {
      break;
    }
644

645 646
    SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1);
    ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
647

648
    if (pr->closed) {
X
Xiaoyu Wang 已提交
649 650
      ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
             isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
651 652 653
      tdListPopHead(pResultRowInfo->openWindow);
      continue;
    }
654

655
    STimeWindow w = pr->win;
X
Xiaoyu Wang 已提交
656
    int32_t     ret =
657 658
        setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx,
                               numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
659 660 661 662 663 664
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

    ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP));

X
Xiaoyu Wang 已提交
665 666
    SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0);
    int64_t     prevTs = *(int64_t*)pTsKey->pData;
667 668
    doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey,
                              RESULT_ROW_END_INTERP, pSup);
669 670

    setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
671
    setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
672

673
    doApplyFunctions(pTaskInfo, pSup->pCtx, &w, &pInfo->twAggSup.timeWindowData, startPos, 0, tsCols,
674 675 676 677 678
                     pBlock->info.rows, numOfExprs, pInfo->order);

    if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
      closeResultRow(pr);
      tdListPopHead(pResultRowInfo->openWindow);
X
Xiaoyu Wang 已提交
679
    } else {  // the remains are can not be closed yet.
680
      break;
681
    }
682
  }
683
}
684

5
54liuyao 已提交
685
typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
686

X
Xiaoyu Wang 已提交
687 688 689
int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) {
  int firstPos = 0, lastPos = num - 1, midPos = -1;
  int numOfRows = 0;
5
54liuyao 已提交
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735

  if (num <= 0) return -1;
  if (order == TSDB_ORDER_DESC) {
    // find the first position which is smaller or equal than the key
    while (1) {
      if (key >= getValuefn(keyList, lastPos)) return lastPos;
      if (key == getValuefn(keyList, firstPos)) return firstPos;
      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }

  } else {
    // find the first position which is bigger or equal than the key
    while (1) {
      if (key <= getValuefn(keyList, firstPos)) return firstPos;
      if (key == getValuefn(keyList, lastPos)) return lastPos;

      if (key > getValuefn(keyList, lastPos)) {
        lastPos = lastPos + 1;
        if (lastPos >= num)
          return -1;
        else
          return lastPos;
      }

      numOfRows = lastPos - firstPos + 1;
      midPos = (numOfRows >> 1) + firstPos;

      if (key < getValuefn(keyList, midPos)) {
        lastPos = midPos - 1;
      } else if (key > getValuefn(keyList, midPos)) {
        firstPos = midPos + 1;
      } else {
        break;
      }
    }
736 737
  }

5
54liuyao 已提交
738 739 740 741
  return midPos;
}

int64_t getReskey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
742
  SArray*     res = (SArray*)data;
5
54liuyao 已提交
743 744 745 746
  SResKeyPos* pos = taosArrayGetP(res, index);
  return *(int64_t*)pos->key;
}

5
54liuyao 已提交
747 748
static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId,
    SArray* pUpdated) {
5
54liuyao 已提交
749
  int32_t size = taosArrayGetSize(pUpdated);
5
54liuyao 已提交
750
  int32_t index = binarySearch(pUpdated, size, ts, TSDB_ORDER_DESC, getReskey);
5
54liuyao 已提交
751 752 753 754
  if (index == -1) {
    index = 0;
  } else {
    TSKEY resTs = getReskey(pUpdated, index);
5
54liuyao 已提交
755
    if (resTs < ts) {
5
54liuyao 已提交
756 757 758 759 760
      index++;
    } else {
      return TSDB_CODE_SUCCESS;
    }
  }
H
Haojun Liao 已提交
761

5
54liuyao 已提交
762 763 764 765 766
  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
  if (newPos == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  newPos->groupId = groupId;
5
54liuyao 已提交
767 768
  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
  *(int64_t*)newPos->key = ts;
X
Xiaoyu Wang 已提交
769
  if (taosArrayInsert(pUpdated, index, &newPos) == NULL) {
5
54liuyao 已提交
770 771 772 773 774
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
775 776 777 778
static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
  return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated);
}

5
54liuyao 已提交
779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
static void removeResult(SArray* pUpdated, TSKEY key) {
  int32_t size = taosArrayGetSize(pUpdated);
  int32_t index = binarySearch(pUpdated, size, key, TSDB_ORDER_DESC, getReskey);
  if (index >= 0 && key == getReskey(pUpdated, index)) {
    taosArrayRemove(pUpdated, index);
  }
}

static void removeResults(SArray* pWins, SArray* pUpdated) {
  int32_t size = taosArrayGetSize(pWins);
  for (int32_t i = 0; i < size; i++) {
    STimeWindow* pW = taosArrayGet(pWins, i);
    removeResult(pUpdated, pW->skey);
  }
}

5
54liuyao 已提交
795
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
796
                            int32_t scanFlag, SArray* pUpdated) {
797
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
798

799
  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
800
  SExprSupp* pSup = &pOperatorInfo->exprSupp;
801

X
Xiaoyu Wang 已提交
802
  int32_t     startPos = 0;
803
  int32_t     numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
804 805 806 807 808
  int64_t*    tsCols = extractTsCol(pBlock, pInfo);
  uint64_t    tableGroupId = pBlock->info.groupId;
  bool        ascScan = (pInfo->order == TSDB_ORDER_ASC);
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;
809 810 811 812

  STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
                                        pInfo->interval.precision, &pInfo->win);

X
Xiaoyu Wang 已提交
813
  int32_t ret =
814 815
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                             numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
816 817 818 819
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
  }

5
54liuyao 已提交
820
  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
821
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
822
      saveResultRow(pResult, tableGroupId, pUpdated);
5
54liuyao 已提交
823
    }
824 825
  }

X
Xiaoyu Wang 已提交
826 827 828
  TSKEY   ekey = ascScan ? win.ekey : win.skey;
  int32_t forwardRows =
      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
829
  ASSERT(forwardRows > 0);
830 831

  // prev time window not interpolation yet.
832
  if (pInfo->timeWindowInterpo) {
833 834
    SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult);
    doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos);
835 836

    // restore current time window
X
Xiaoyu Wang 已提交
837
    ret =
838 839
        setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                               numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
840 841 842 843
    if (ret != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

844
    // window start key interpolation
845
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
846
  }
847 848

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
849
  doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
850 851 852
                   pBlock->info.rows, numOfOutput, pInfo->order);

  doCloseWindow(pResultRowInfo, pInfo, pResult);
853 854 855

  STimeWindow nextWin = win;
  while (1) {
856
    int32_t prevEndPos = forwardRows - 1 + startPos;
857
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->order);
858 859 860 861 862
    if (startPos < 0) {
      break;
    }

    // null data, failed to allocate more memory buffer
X
Xiaoyu Wang 已提交
863
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
864
                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
865
                                          &pInfo->aggSup, pTaskInfo);
866 867 868 869
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

5
54liuyao 已提交
870
    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
871
      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
872
        saveResultRow(pResult, tableGroupId, pUpdated);
5
54liuyao 已提交
873
      }
874 875
    }

X
Xiaoyu Wang 已提交
876
    ekey = ascScan ? nextWin.ekey : nextWin.skey;
877
    forwardRows =
878
        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
879 880

    // window start(end) key interpolation
881
    doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup);
882 883

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
884
    doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
X
Xiaoyu Wang 已提交
885
                     tsCols, pBlock->info.rows, numOfOutput, pInfo->order);
886
    doCloseWindow(pResultRowInfo, pInfo, pResult);
887 888 889
  }

  if (pInfo->timeWindowInterpo) {
890
    saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
891
  }
892 893 894 895 896 897 898 899 900 901 902 903
}

void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
  // current result is done in computing final results.
  if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
    closeResultRow(pResult);
    tdListPopHead(pResultRowInfo->openWindow);
  }
}

SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult) {
  SResultRowPosition pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
X
Xiaoyu Wang 已提交
904
  SListNode*         pn = tdListGetTail(pResultRowInfo->openWindow);
905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929
  if (pn == NULL) {
    tdListAppend(pResultRowInfo->openWindow, &pos);
    return pos;
  }

  SResultRowPosition* px = (SResultRowPosition*)pn->data;
  if (px->pageId != pos.pageId || px->offset != pos.offset) {
    tdListAppend(pResultRowInfo->openWindow, &pos);
  }

  return pos;
}

int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
  TSKEY* tsCols = NULL;
  if (pBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;

    if (tsCols != NULL) {
      blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
    }
  }

  return tsCols;
930 931 932 933 934 935 936
}

static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
  if (OPTR_IS_OPENED(pOperator)) {
    return TSDB_CODE_SUCCESS;
  }

L
Liu Jicong 已提交
937
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
938
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
939
  SExprSupp*                pSup = &pOperator->exprSupp;
940

941 942
  int32_t scanFlag = MAIN_SCAN;

X
Xiaoyu Wang 已提交
943
  int64_t        st = taosGetTimestampUs();
944 945 946
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
947
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
948 949 950 951
    if (pBlock == NULL) {
      break;
    }

952 953
    getTableScanInfo(pOperator, &pInfo->order, &scanFlag);

954
    // the pDataBlock are always the same one, no need to call this again
955
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, scanFlag, true);
H
Haojun Liao 已提交
956
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL);
957 958

#if 0  // test for encode/decode result info
959
    if(pOperator->fpSet.encodeResultRow){
960 961 962
      char *result = NULL;
      int32_t length = 0;
      SAggSupporter   *pSup = &pInfo->aggSup;
963
      pOperator->fpSet.encodeResultRow(pOperator, &result, &length);
964 965
      taosHashClear(pSup->pResultRowHashTable);
      pInfo->binfo.resultRowInfo.size = 0;
966
      pOperator->fpSet.decodeResultRow(pOperator, result);
967 968 969 970 971 972 973 974
      if(result){
        taosMemoryFree(result);
      }
    }
#endif
  }

  closeAllResultRows(&pInfo->binfo.resultRowInfo);
975
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->order);
976
  OPTR_SET_OPENED(pOperator);
977 978

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
979 980 981
  return TSDB_CODE_SUCCESS;
}

982 983 984 985 986 987 988 989 990 991 992 993
static bool compareVal(const char* v, const SStateKeys* pKey) {
  if (IS_VAR_DATA_TYPE(pKey->type)) {
    if (varDataLen(v) != varDataLen(pKey->pData)) {
      return false;
    } else {
      return strncmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0;
    }
  } else {
    return memcmp(pKey->pData, v, pKey->bytes) == 0;
  }
}

994
static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
L
Liu Jicong 已提交
995
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
996
  SExprSupp* pSup = &pOperator->exprSupp;
997

998
  SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
999 1000 1001
  int64_t          gid = pBlock->info.groupId;

  bool    masterScan = true;
1002
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1003 1004
  int16_t bytes = pStateColInfoData->info.bytes;

1005
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1006 1007 1008 1009 1010
  TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

1011
  struct SColumnDataAgg* pAgg = NULL;
1012
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
X
Xiaoyu Wang 已提交
1013
    pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
1014
    if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
1015 1016 1017 1018 1019 1020
      continue;
    }

    char* val = colDataGetData(pStateColInfoData, j);

    if (!pInfo->hasKey) {
1021 1022 1023 1024 1025 1026 1027
      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }

1028 1029 1030 1031
      pInfo->hasKey = true;

      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
1032
    } else if (compareVal(val, &pInfo->stateKey)) {
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
      doKeepTuple(pRowSup, tsList[j]);
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // a new state window started
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
      int32_t ret =
1045 1046
          setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1047 1048 1049 1050 1051
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
      }

      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1052
      doApplyFunctions(pTaskInfo, pSup->pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1053 1054 1055 1056 1057
                       pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);

      // here we start a new session window
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
1058 1059 1060 1061 1062 1063 1064

      // todo extract method
      if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
        varDataCopy(pInfo->stateKey.pData, val);
      } else {
        memcpy(pInfo->stateKey.pData, val, bytes);
      }
1065 1066 1067 1068 1069 1070
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
  int32_t ret =
1071 1072
      setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pSup->pCtx,
                             numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1073 1074 1075 1076 1077
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
1078
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1079 1080 1081
                   pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
}

1082
static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
1083 1084 1085 1086 1087
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SStateWindowOperatorInfo* pInfo = pOperator->info;
1088 1089

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
1090 1091
  SExprSupp* pSup = &pOperator->exprSupp;

1092
  SOptrBasicInfo* pBInfo = &pInfo->binfo;
1093 1094

  if (pOperator->status == OP_RES_TO_RETURN) {
1095
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1096
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1097 1098 1099 1100 1101 1102 1103
      doSetOperatorCompleted(pOperator);
      return NULL;
    }

    return pBInfo->pRes;
  }

1104
  int32_t order = TSDB_ORDER_ASC;
1105
  int64_t st = taosGetTimestampUs();
1106 1107 1108

  SOperatorInfo* downstream = pOperator->pDownstream[0];
  while (1) {
1109
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1110 1111 1112 1113
    if (pBlock == NULL) {
      break;
    }

1114
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1115 1116
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1117 1118 1119
    doStateWindowAggImpl(pOperator, pInfo, pBlock);
  }

X
Xiaoyu Wang 已提交
1120
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
1121

1122 1123 1124
  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pBInfo->resultRowInfo);

1125
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1126
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1127
  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1128
  if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1129 1130 1131
    doSetOperatorCompleted(pOperator);
  }

1132 1133 1134
  size_t rows = pBInfo->pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1135
  return (rows == 0) ? NULL : pBInfo->pRes;
1136 1137
}

1138
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
1139
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
L
Liu Jicong 已提交
1140
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1141 1142 1143 1144 1145 1146 1147 1148

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSDataBlock* pBlock = pInfo->binfo.pRes;

  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
1149
    return pOperator->fpSet.getStreamResFn(pOperator);
1150 1151 1152 1153 1154 1155 1156
  } else {
    pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
    if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
      return NULL;
    }

    blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
1157
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1158

1159
    if (pBlock->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1160 1161 1162
      doSetOperatorCompleted(pOperator);
    }

1163 1164 1165
    size_t rows = pBlock->info.rows;
    pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1166
    return (rows == 0) ? NULL : pBlock;
1167 1168 1169 1170
  }
}

// todo merged with the build group result.
1171
static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
1172
                                  int32_t* rowEntryInfoOffset) {
1173 1174 1175 1176 1177 1178 1179
  size_t num = taosArrayGetSize(pUpdateList);

  for (int32_t i = 0; i < num; ++i) {
    SResKeyPos* pPos = taosArrayGetP(pUpdateList, i);

    SFilePage*  bufPage = getBufPage(pBuf, pPos->pos.pageId);
    SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
1180

1181
    for (int32_t j = 0; j < numOfOutput; ++j) {
1182
      SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, j, rowEntryInfoOffset);
1183 1184
      if (pRow->numOfRows < pEntry->numOfRes) {
        pRow->numOfRows = pEntry->numOfRes;
1185 1186 1187 1188 1189 1190
      }
    }

    releaseBufPage(pBuf, bufPage);
  }
}
5
54liuyao 已提交
1191
static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) {
L
Liu Jicong 已提交
1192
  for (int i = 0; i < num; i++) {
5
54liuyao 已提交
1193 1194
    if (type == STREAM_INVERT) {
      fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
L
Liu Jicong 已提交
1195
    } else if (type == STREAM_NORMAL) {
5
54liuyao 已提交
1196 1197 1198 1199
      fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet));
    }
  }
}
5
54liuyao 已提交
1200

1201
void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp *pSup, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
1202
  SResultRow*     pResult = getResultRowByPos(pResultBuf, p1);
1203
  SqlFunctionCtx* pCtx = pSup->pCtx;
5
54liuyao 已提交
1204
  for (int32_t i = 0; i < numOfOutput; ++i) {
1205
    pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216
    struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
    if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
      continue;
    }
    pResInfo->initialized = false;
    if (pCtx[i].functionId != -1) {
      pCtx[i].fpSet.init(&pCtx[i], pResInfo);
    }
  }
}

1217
void doClearWindow(SAggSupporter* pAggSup, SExprSupp *pSup, char* pData, int16_t bytes, uint64_t groupId,
X
Xiaoyu Wang 已提交
1218
                   int32_t numOfOutput) {
1219
  SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId);
5
54liuyao 已提交
1220
  SResultRowPosition* p1 =
1221
      (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
1222 1223 1224 1225
  if (!p1) {
    // window has been closed
    return;
  }
1226
  doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput);
5
54liuyao 已提交
1227 1228
}

1229
static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t tsIndex,
X
Xiaoyu Wang 已提交
1230
                           int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) {
5
54liuyao 已提交
1231
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
1232 1233
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
  int32_t          step = 0;
5
54liuyao 已提交
1234 1235 1236
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
X
Xiaoyu Wang 已提交
1237 1238
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL);
    step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
1239
    doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput);
1240 1241 1242
    if (pUpWins) {
      taosArrayPush(pUpWins, &win);
    }
5
54liuyao 已提交
1243 1244
  }
}
1245

5
54liuyao 已提交
1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) {
  void*  pIte = NULL;
  size_t keyLen = 0;
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    void*    key = taosHashGetKey(pIte, &keyLen);
    uint64_t groupId = *(uint64_t*)key;
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
    TSKEY          ts = *(int64_t*)((char*)key + sizeof(uint64_t));
    SResultRowPosition* pPos = (SResultRowPosition*)pIte;
    int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
1263 1264 1265 1266
bool isCloseWindow(STimeWindow *pWin, STimeWindowAggSupp* pSup) {
  return pWin->ekey < pSup->maxTs - pSup->waterMark;
}

X
Xiaoyu Wang 已提交
1267 1268 1269
static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
                                   SArray* closeWins) {
  void*  pIte = NULL;
5
54liuyao 已提交
1270
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
1271 1272 1273
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    void*    key = taosHashGetKey(pIte, &keyLen);
    uint64_t groupId = *(uint64_t*)key;
5
54liuyao 已提交
1274
    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
X
Xiaoyu Wang 已提交
1275
    TSKEY          ts = *(int64_t*)((char*)key + sizeof(uint64_t));
5
54liuyao 已提交
1276 1277
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
X
Xiaoyu Wang 已提交
1278
    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL);
5
54liuyao 已提交
1279
    if (isCloseWindow(&win, pSup)) {
5
54liuyao 已提交
1280 1281
      char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
      SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
1282
      taosHashRemove(pHashMap, keyBuf, keyLen);
5
54liuyao 已提交
1283 1284 1285 1286 1287 1288
      SResultRowPosition* pPos = (SResultRowPosition*)pIte;
      if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
        int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins);
        if (code != TSDB_CODE_SUCCESS) {
          return code;
        }
5
54liuyao 已提交
1289 1290 1291 1292 1293 1294
      }
    }
  }
  return TSDB_CODE_SUCCESS;
}

1295
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
1296
  SIntervalAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
1297
  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
1298 1299

  pInfo->order = TSDB_ORDER_ASC;
1300
  SExprSupp* pSup = &pOperator->exprSupp;
1301 1302 1303 1304 1305 1306

  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
1307
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1308
    if (pInfo->binfo.pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1309 1310 1311 1312 1313 1314 1315
      pOperator->status = OP_EXEC_DONE;
    }
    return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
  }

  SOperatorInfo* downstream = pOperator->pDownstream[0];

5
54liuyao 已提交
1316
  SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);
1317
  while (1) {
1318
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1319 1320 1321 1322
    if (pBlock == NULL) {
      break;
    }

5
54liuyao 已提交
1323
    if (pBlock->info.type == STREAM_REPROCESS) {
1324
      doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, NULL);
1325
      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
5
54liuyao 已提交
1326
      continue;
1327
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
1328 1329
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
      continue;
5
54liuyao 已提交
1330
    }
1331

1332 1333 1334
    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
    // caller. Note that all the time window are not close till now.
    // the pDataBlock are always the same one, no need to call this again
1335
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
1336
    if (pInfo->invertible) {
1337
      setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type);
1338 1339
    }

5
54liuyao 已提交
1340
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
H
Haojun Liao 已提交
1341
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated);
1342
  }
5
54liuyao 已提交
1343
  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
1344

1345
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
1346 1347
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
1348
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1349 1350 1351 1352 1353 1354 1355 1356

  pOperator->status = OP_RES_TO_RETURN;

  return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}

static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
1357
  cleanupBasicInfo(&pInfo->binfo);
1358 1359 1360 1361
  taosMemoryFreeClear(pInfo->stateKey.pData);
}

void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) {
1362
  SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
1363
  cleanupBasicInfo(&pInfo->binfo);
1364 1365 1366
  cleanupAggSup(&pInfo->aggSup);
}

5
54liuyao 已提交
1367
void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
1368
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
1369
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
1370
  cleanupAggSup(&pInfo->aggSup);
1371 1372 1373 1374 1375 1376 1377 1378 1379
  if (pInfo->pChildren) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
      destroyIntervalOperatorInfo(pChildOp->info, numOfOutput);
      taosMemoryFreeClear(pChildOp->info);
      taosMemoryFreeClear(pChildOp);
    }
  }
1380
  nodesDestroyNode((SNode*)pInfo->pPhyNode);
5
54liuyao 已提交
1381 1382
}

1383
static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
5
54liuyao 已提交
1384 1385 1386 1387 1388 1389 1390 1391
  for (int32_t i = 0; i < numOfCols; i++) {
    if (!fmIsInvertible(pFCtx[i].functionId)) {
      return false;
    }
  }
  return true;
}

1392
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
1393 1394
  // the primary timestamp column
  bool needed = false;
1395 1396
  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
1397

X
Xiaoyu Wang 已提交
1398
  {  // ts column
1399 1400
    SColumn c = {0};
    c.colId = 1;
1401
    c.slotId = pInfo->primaryTsIndex;
1402 1403
    c.type = TSDB_DATA_TYPE_TIMESTAMP;
    c.bytes = sizeof(int64_t);
1404
    taosArrayPush(pInfo->pInterpCols, &c);
1405 1406

    SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1407 1408 1409 1410
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = true;  // to denote no value is assigned yet
    key.pData = taosMemoryCalloc(1, c.bytes);
1411
    taosArrayPush(pInfo->pPrevValues, &key);
1412 1413
  }

X
Xiaoyu Wang 已提交
1414
  for (int32_t i = 0; i < numOfCols; ++i) {
1415 1416
    SExprInfo* pExpr = pCtx[i].pExpr;

H
Haojun Liao 已提交
1417
    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
1418 1419 1420
      SFunctParam* pParam = &pExpr->base.pParam[0];

      SColumn c = *pParam->pCol;
1421
      taosArrayPush(pInfo->pInterpCols, &c);
1422 1423 1424
      needed = true;

      SGroupKeys key = {0};
X
Xiaoyu Wang 已提交
1425 1426
      key.bytes = c.bytes;
      key.type = c.type;
1427
      key.isNull = false;
X
Xiaoyu Wang 已提交
1428
      key.pData = taosMemoryCalloc(1, c.bytes);
1429
      taosArrayPush(pInfo->pPrevValues, &key);
1430 1431 1432 1433 1434 1435
    }
  }

  return needed;
}

1436 1437 1438 1439 1440 1441
void increaseTs(SqlFunctionCtx* pCtx) {
  if (pCtx[0].pExpr->pExpr->_function.pFunctNode->funcType == FUNCTION_TYPE_WSTARTTS) {
    pCtx[0].increase = true;
  }
}

1442 1443
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                          SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
1444
                                          STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream) {
1445
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1446
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1447 1448 1449 1450
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

X
Xiaoyu Wang 已提交
1451 1452 1453
  pInfo->win = pTaskInfo->window;
  pInfo->order = TSDB_ORDER_ASC;
  pInfo->interval = *pInterval;
L
Liu Jicong 已提交
1454
  pInfo->execModel = pTaskInfo->execModel;
X
Xiaoyu Wang 已提交
1455
  pInfo->twAggSup = *pTwAggSupp;
1456

1457 1458
  pInfo->primaryTsIndex = primaryTsSlotId;

1459 1460
  SExprSupp* pSup = &pOperator->exprSupp;

1461 1462
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);
1463

1464 1465 1466
  int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);

1467 1468
  if (isStream) {
    ASSERT(numOfCols > 0);
1469
    increaseTs(pSup->pCtx);
1470
  }
1471

1472
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
1473

1474
  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
X
Xiaoyu Wang 已提交
1475
  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
1476

1477
  pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, pInfo);
1478 1479
  if (pInfo->timeWindowInterpo) {
    pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
H
Haojun Liao 已提交
1480 1481 1482
    if (pInfo->binfo.resultRowInfo.openWindow == NULL) {
      goto _error;
    }
1483 1484
  }

1485
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1486

X
Xiaoyu Wang 已提交
1487 1488 1489 1490
  pOperator->name = "TimeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
1491
  pOperator->exprSupp.pExprInfo = pExprInfo;
X
Xiaoyu Wang 已提交
1492
  pOperator->pTaskInfo = pTaskInfo;
1493
  pOperator->exprSupp.numOfExprs = numOfCols;
X
Xiaoyu Wang 已提交
1494
  pOperator->info = pInfo;
1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505

  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL,
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1506
_error:
1507 1508 1509 1510 1511 1512 1513 1514 1515
  destroyIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                                SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
wmmhello's avatar
wmmhello 已提交
1516
                                                STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
1517
  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
L
Liu Jicong 已提交
1518
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

  pInfo->order = TSDB_ORDER_ASC;
  pInfo->interval = *pInterval;
  pInfo->execModel = OPTR_EXEC_MODEL_STREAM;
  pInfo->win = pTaskInfo->window;
  pInfo->twAggSup = *pTwAggSupp;
  pInfo->primaryTsIndex = primaryTsSlotId;

  int32_t numOfRows = 4096;
  size_t  keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

  initResultSizeInfo(pOperator, numOfRows);
1534 1535
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);
1536 1537
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);

wmmhello's avatar
wmmhello 已提交
1538
  if (code != TSDB_CODE_SUCCESS) {
1539 1540 1541
    goto _error;
  }

1542
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1543 1544

  pOperator->name = "StreamTimeIntervalAggOperator";
X
Xiaoyu Wang 已提交
1545
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
1546
  pOperator->blocking = true;
1547
  pOperator->status = OP_NOT_OPENED;
1548
  pOperator->exprSupp.pExprInfo = pExprInfo;
1549
  pOperator->pTaskInfo = pTaskInfo;
1550
  pOperator->exprSupp.numOfExprs = numOfCols;
1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562
  pOperator->info = pInfo;

  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL,
                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

L
Liu Jicong 已提交
1563
_error:
1564 1565 1566 1567 1568 1569 1570 1571 1572 1573
  destroyIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

// todo handle multiple tables cases.
static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
1574
  SExprSupp* pSup = &pOperator->exprSupp;
1575

1576
  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
1577 1578

  bool    masterScan = true;
1579
  int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611
  int64_t gid = pBlock->info.groupId;

  int64_t gap = pInfo->gap;

  if (!pInfo->reptScan) {
    pInfo->reptScan = true;
    pInfo->winSup.prevTs = INT64_MIN;
  }

  SWindowRowsSup* pRowSup = &pInfo->winSup;
  pRowSup->numOfRows = 0;

  // In case of ascending or descending order scan data, only one time window needs to be kepted for each table.
  TSKEY* tsList = (TSKEY*)pColInfoData->pData;
  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
    if (pInfo->winSup.prevTs == INT64_MIN) {
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
    } else if (tsList[j] - pRowSup->prevTs <= gap && (tsList[j] - pRowSup->prevTs) >= 0) {
      // The gap is less than the threshold, so it belongs to current session window that has been opened already.
      doKeepTuple(pRowSup, tsList[j]);
      if (j == 0 && pRowSup->startRowIndex != 0) {
        pRowSup->startRowIndex = 0;
      }
    } else {  // start a new session window
      SResultRow* pResult = NULL;

      // keep the time window for the closed time window.
      STimeWindow window = pRowSup->win;

      pRowSup->win.ekey = pRowSup->win.skey;
      int32_t ret =
1612 1613
          setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1614 1615 1616 1617 1618 1619
      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
      }

      // pInfo->numOfRows data belong to the current session window
      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
1620
      doApplyFunctions(pTaskInfo, pSup->pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631
                       pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);

      // here we start a new session window
      doKeepNewWindowStartInfo(pRowSup, tsList, j);
      doKeepTuple(pRowSup, tsList[j]);
    }
  }

  SResultRow* pResult = NULL;
  pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
  int32_t ret =
1632 1633
      setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pSup->pCtx,
                             numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
1634 1635 1636 1637 1638
  if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
  }

  updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
1639
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
1640 1641 1642
                   pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
}

1643
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
1644 1645 1646 1647 1648 1649
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  SSessionAggOperatorInfo* pInfo = pOperator->info;
  SOptrBasicInfo*          pBInfo = &pInfo->binfo;
1650
  SExprSupp* pSup = &pOperator->exprSupp;
1651 1652

  if (pOperator->status == OP_RES_TO_RETURN) {
1653
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1654
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1655 1656 1657 1658 1659 1660 1661
      doSetOperatorCompleted(pOperator);
      return NULL;
    }

    return pBInfo->pRes;
  }

1662 1663 1664
  int64_t st = taosGetTimestampUs();
  int32_t order = TSDB_ORDER_ASC;

1665 1666 1667
  SOperatorInfo* downstream = pOperator->pDownstream[0];

  while (1) {
1668
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1669 1670 1671 1672 1673
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1674
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
1675 1676
    blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);

1677 1678 1679
    doSessionWindowAggImpl(pOperator, pInfo, pBlock);
  }

1680 1681
  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;

1682 1683 1684 1685
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
  closeAllResultRows(&pBInfo->resultRowInfo);

1686
  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
1687
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
1688
  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
1689
  if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
1690 1691 1692
    doSetOperatorCompleted(pOperator);
  }

1693 1694 1695
  size_t rows = pBInfo->pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;

X
Xiaoyu Wang 已提交
1696
  return (rows == 0) ? NULL : pBInfo->pRes;
1697 1698
}

H
Haojun Liao 已提交
1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock) {
  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for(int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData*  pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

    // null data should not be kept since it can not be used to perform interpolation
    if (!colDataIsNull_s(pColInfoData, i)) {
      SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);

      pkey->isNull = false;
      char* val = colDataGetData(pColInfoData, i);
      memcpy(pkey->pData, val, pkey->bytes);
    }
  }
}

H
Haojun Liao 已提交
1715
static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
1716 1717 1718 1719 1720
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

  STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
H
Haojun Liao 已提交
1721
  SSDataBlock* pResBlock = pSliceInfo->binfo.pRes;
1722
  SExprSupp* pSup = &pOperator->exprSupp;
H
Haojun Liao 已提交
1723

H
Haojun Liao 已提交
1724 1725
//  if (pOperator->status == OP_RES_TO_RETURN) {
//    //    doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes);
1726
//    if (pResBlock->info.rows == 0 || !hasDataInGroupInfo(&pSliceInfo->groupResInfo)) {
H
Haojun Liao 已提交
1727 1728 1729 1730 1731
//      doSetOperatorCompleted(pOperator);
//    }
//
//    return pResBlock;
//  }
1732

H
Haojun Liao 已提交
1733 1734
  int32_t order = TSDB_ORDER_ASC;
  SInterval* pInterval = &pSliceInfo->interval;
1735 1736
  SOperatorInfo* downstream = pOperator->pDownstream[0];

H
Haojun Liao 已提交
1737
  int32_t numOfRows = 0;
1738
  while (1) {
1739
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
1740 1741 1742 1743 1744
    if (pBlock == NULL) {
      break;
    }

    // the pDataBlock are always the same one, no need to call this again
1745
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true);
H
Haojun Liao 已提交
1746 1747 1748 1749 1750 1751

    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, 0);
    for(int32_t i = 0; i < pBlock->info.rows; ++i) {
      int64_t ts = *(int64_t*) colDataGetData(pTsCol, i);

      if (ts == pSliceInfo->current) {
1752 1753
        for(int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
          SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
H
Haojun Liao 已提交
1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764
          int32_t dstSlot = pExprInfo->base.resSchema.slotId;
          int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;

          SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
          SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, dstSlot);

          char* v = colDataGetData(pSrc, i);
          colDataAppend(pDst, numOfRows, v, false);
        }

        numOfRows += 1;
H
Haojun Liao 已提交
1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775

        pSliceInfo->current += taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
        if (pSliceInfo->current > pSliceInfo->win.ekey) {
          doSetOperatorCompleted(pOperator);
          break;
        }
      } else if (ts < pSliceInfo->current) {
        if (i != pBlock->info.window.ekey) {
          int64_t nextTs = *(int64_t*) colDataGetData(pTsCol, i + 1);
          if (nextTs > pSliceInfo->current) {
            // output the result
1776 1777
            for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
              SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j];
H
Haojun Liao 已提交
1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
              int32_t    dstSlot = pExprInfo->base.resSchema.slotId;
              int32_t    srcSlot = pExprInfo->base.pParam[0].pCol->slotId;

              SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot);
              SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, dstSlot);

              switch (pSliceInfo->fillType) {
                case TSDB_FILL_NULL:
                  colDataAppendNULL(pDst, numOfRows);
                  break;

                case TSDB_FILL_SET_VALUE: {
                  SVariant* pVar = &pSliceInfo->pFillColInfo[i].fillVal;

                  if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
                    float v = 0;
                    GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
                    colDataAppend(pDst, numOfRows, (char*)&v, false);
                  } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
                    double v = 0;
                    GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
                    colDataAppend(pDst, numOfRows, (char*)&v, false);
                  } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
                    int64_t v = 0;
                    GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
                    colDataAppend(pDst, numOfRows, (char*)&v, false);
                  }
                }
                break;

                case TSDB_FILL_LINEAR:
#if 0
                if (pCtx->start.key == INT64_MIN || pCtx->start.key > pCtx->startTs
                    || pCtx->end.key == INT64_MIN || pCtx->end.key < pCtx->startTs) {
//                  goto interp_exit;
                }

              double v1 = -1, v2 = -1;
              GET_TYPED_DATA(v1, double, pCtx->inputType, &pCtx->start.val);
              GET_TYPED_DATA(v2, double, pCtx->inputType, &pCtx->end.val);

              SPoint point1 = {.key = ts, .val = &v1};
              SPoint point2 = {.key = nextTs, .val = &v2};
              SPoint point  = {.key = pCtx->startTs, .val = pCtx->pOutput};

              int32_t srcType = pCtx->inputType;
              if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) {
                setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
              } else {
                bool exceedMax = false, exceedMin = false;
                taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE, &exceedMax, &exceedMin);
                if (exceedMax || exceedMin) {
                  __compar_fn_t func = getComparFunc((int32_t)pCtx->inputType, 0);
                  if (func(&pCtx->start.val, &pCtx->end.val) <= 0) {
                    COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->start.val : &pCtx->end.val);
                  } else {
                    COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->end.val : &pCtx->start.val);
                  }
                }
              }
#endif
                  break;

                case TSDB_FILL_PREV: {
                  SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
                  colDataAppend(pDst, numOfRows, pkey->pData, false);
                } break;

                case TSDB_FILL_NEXT: {
                } break;

                case TSDB_FILL_NONE:
                default:
                  break;
              }

              pSliceInfo->current +=
                  taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
              if (pSliceInfo->current > pSliceInfo->win.ekey) {
                doSetOperatorCompleted(pOperator);
                break;
              }
H
Haojun Liao 已提交
1860 1861
            }
          } else {
H
Haojun Liao 已提交
1862
            // ignore current row, and do nothing
H
Haojun Liao 已提交
1863 1864
          }
        } else {  // it is the last row of current block
H
Haojun Liao 已提交
1865
          doKeepPrevRows(pSliceInfo, pBlock);
H
Haojun Liao 已提交
1866 1867 1868
        }
      }
    }
1869 1870 1871 1872
  }

  // restore the value
  setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
H
Haojun Liao 已提交
1873
  if (pResBlock->info.rows == 0) {
1874 1875 1876
    pOperator->status = OP_EXEC_DONE;
  }

H
Haojun Liao 已提交
1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904
  return pResBlock->info.rows == 0 ? NULL : pResBlock;
}

static int32_t initTimesliceInfo(STimeSliceOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfCols) {
  pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
  pInfo->pCols = taosArrayInit(4, sizeof(SColumn));

  if (pInfo->pPrevRow == NULL || pInfo->pCols == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  for (int32_t i = 0; i < numOfCols; ++i) {
    SExprInfo* pExpr = pCtx[i].pExpr;

    SFunctParam* pParam = &pExpr->base.pParam[0];

    SColumn c = *pParam->pCol;
    taosArrayPush(pInfo->pCols, &c);

    SGroupKeys key = {0};
    key.bytes = c.bytes;
    key.type = c.type;
    key.isNull = false;
    key.pData = taosMemoryCalloc(1, c.bytes);
    taosArrayPush(pInfo->pPrevRow, &key);
  }

  return TSDB_CODE_SUCCESS;
1905 1906 1907
}

SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
H
Haojun Liao 已提交
1908
                                           SSDataBlock* pResultBlock, const SNodeListNode* pValNode, SExecTaskInfo* pTaskInfo) {
1909 1910 1911 1912 1913 1914
  STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pOperator == NULL || pInfo == NULL) {
    goto _error;
  }

1915 1916 1917
  SExprSupp* pSup = &pOperator->exprSupp;

  int32_t code = initTimesliceInfo(pInfo, pSup->pCtx, numOfCols);
H
Haojun Liao 已提交
1918 1919 1920 1921
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

1922
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
H
Haojun Liao 已提交
1923
  pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfCols, pValNode);
1924

H
Haojun Liao 已提交
1925
  pInfo->binfo.pRes     = pResultBlock;
H
Haojun Liao 已提交
1926 1927

  pOperator->name       = "TimeSliceOperator";
1928
  //  pOperator->operatorType = OP_AllTimeWindow;
H
Haojun Liao 已提交
1929 1930
  pOperator->blocking   = true;
  pOperator->status     = OP_NOT_OPENED;
1931 1932
  pOperator->exprSupp.pExprInfo      = pExprInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
H
Haojun Liao 已提交
1933 1934
  pOperator->info       = pInfo;
  pOperator->pTaskInfo  = pTaskInfo;
1935

H
Haojun Liao 已提交
1936
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyBasicOperatorInfo,
1937 1938
                                         NULL, NULL, NULL);

H
Haojun Liao 已提交
1939
  code = appendDownstream(pOperator, &downstream, 1);
1940 1941
  return pOperator;

L
Liu Jicong 已提交
1942
_error:
1943 1944 1945 1946 1947 1948 1949
  taosMemoryFree(pInfo);
  taosMemoryFree(pOperator);
  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
  return NULL;
}

SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
1950
                                             SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId,
1951
                                             SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo) {
1952 1953 1954 1955 1956 1957
  SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

1958 1959 1960 1961 1962 1963 1964 1965
  pInfo->stateCol = *pStateKeyCol;
  pInfo->stateKey.type = pInfo->stateCol.type;
  pInfo->stateKey.bytes = pInfo->stateCol.bytes;
  pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
  if (pInfo->stateKey.pData == NULL) {
    goto _error;
  }

1966 1967 1968
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

  initResultSizeInfo(pOperator, 4096);
1969 1970 1971
  initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExpr, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);

1972
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
1973

L
Liu Jicong 已提交
1974
  pInfo->twAggSup = *pTwAggSup;
1975 1976
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

X
Xiaoyu Wang 已提交
1977 1978
  pInfo->tsSlotId = tsSlotId;
  pOperator->name = "StateWindowOperator";
1979
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE;
X
Xiaoyu Wang 已提交
1980 1981
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
1982 1983
  pOperator->exprSupp.pExprInfo = pExpr;
  pOperator->exprSupp.numOfExprs = numOfCols;
X
Xiaoyu Wang 已提交
1984 1985
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
1986 1987 1988 1989 1990 1991 1992

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL,
                                         destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);

  int32_t code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

L
Liu Jicong 已提交
1993
_error:
1994 1995 1996 1997 1998 1999
  pTaskInfo->code = TSDB_CODE_SUCCESS;
  return NULL;
}

void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) {
  SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param;
2000
  cleanupBasicInfo(&pInfo->binfo);
2001 2002 2003
}

SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
L
Liu Jicong 已提交
2004 2005
                                            SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId,
                                            STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
2006 2007 2008 2009 2010 2011
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

2012 2013
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);
2014

2015
  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
2016 2017 2018 2019
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

2020 2021
  initBasicInfo(&pInfo->binfo, pResBlock);

2022
  pInfo->twAggSup = *pTwAggSupp;
2023
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
2024 2025
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

L
Liu Jicong 已提交
2026 2027 2028 2029 2030 2031
  pInfo->tsSlotId = tsSlotId;
  pInfo->gap = gap;
  pInfo->binfo.pRes = pResBlock;
  pInfo->winSup.prevTs = INT64_MIN;
  pInfo->reptScan = false;
  pOperator->name = "SessionWindowAggOperator";
2032
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
2033
  pOperator->blocking = true;
L
Liu Jicong 已提交
2034
  pOperator->status = OP_NOT_OPENED;
2035 2036
  pOperator->exprSupp.pExprInfo = pExprInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
L
Liu Jicong 已提交
2037
  pOperator->info = pInfo;
2038 2039 2040 2041 2042 2043 2044 2045

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL,
                                         destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  pOperator->pTaskInfo = pTaskInfo;

  code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

L
Liu Jicong 已提交
2046
_error:
2047 2048 2049 2050 2051 2052 2053 2054
  if (pInfo != NULL) {
    destroySWindowOperatorInfo(pInfo, numOfCols);
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
L
Liu Jicong 已提交
2055
}
5
54liuyao 已提交
2056

5
54liuyao 已提交
2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076
bool isFinalInterval(SStreamFinalIntervalOperatorInfo* pInfo) { return pInfo->pChildren != NULL; }

void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
                      SExecTaskInfo* pTaskInfo) {
  for (int32_t k = 0; k < numOfOutput; ++k) {
    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
      continue;
    }
    int32_t code = TSDB_CODE_SUCCESS;
    if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
      code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
      if (code != TSDB_CODE_SUCCESS) {
        qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
        pTaskInfo->code = code;
        longjmp(pTaskInfo->env, code);
      }
    }
  }
}

2077
static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray, int32_t groupId,
5
54liuyao 已提交
2078 2079 2080 2081 2082 2083
                                  int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
  int32_t size = taosArrayGetSize(pWinArray);
  ASSERT(pInfo->pChildren);
  for (int32_t i = 0; i < size; i++) {
    STimeWindow* pParentWin = taosArrayGet(pWinArray, i);
    SResultRow*  pCurResult = NULL;
2084 2085
    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, pParentWin, true, &pCurResult, 0, pSup->pCtx,
                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2086 2087 2088 2089
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
    for (int32_t j = 0; j < numOfChildren; j++) {
      SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
2090 2091
      SExprSupp* pChildSup = &pChildOp->exprSupp;

5
54liuyao 已提交
2092
      SResultRow*               pChResult = NULL;
2093 2094 2095
      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, pParentWin, true, &pChResult, 0, pChildSup->pCtx,
                             pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup, pTaskInfo);
      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo);
5
54liuyao 已提交
2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107
    }
  }
}

bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
  SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId);
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable,
      pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(int64_t)));
  return p1 == NULL;
}

static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
S
shenglian zhou 已提交
2108
                           SArray* pUpdated) {
5
54liuyao 已提交
2109
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
X
Xiaoyu Wang 已提交
2110 2111
  SResultRowInfo*                   pResultRowInfo = &(pInfo->binfo.resultRowInfo);
  SExecTaskInfo*                    pTaskInfo = pOperatorInfo->pTaskInfo;
2112 2113
  SExprSupp*                        pSup = &pOperatorInfo->exprSupp;
  int32_t                           numOfOutput = pSup->numOfExprs;
X
Xiaoyu Wang 已提交
2114 2115 2116 2117 2118
  int32_t                           step = 1;
  bool                              ascScan = true;
  TSKEY*                            tsCols = NULL;
  SResultRow*                       pResult = NULL;
  int32_t                           forwardRows = 0;
5
54liuyao 已提交
2119 2120 2121 2122

  if (pSDataBlock->pDataBlock != NULL) {
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
2123
  } else {
S
shenglian zhou 已提交
2124
    return;
5
54liuyao 已提交
2125
  }
5
54liuyao 已提交
2126

X
Xiaoyu Wang 已提交
2127 2128 2129 2130
  int32_t     startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
  STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
                                            pInfo->interval.precision, NULL);
5
54liuyao 已提交
2131
  while (1) {
5
54liuyao 已提交
2132 2133 2134 2135
    if (isFinalInterval(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) &&
        isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup)) {
      SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
      taosArrayPush(pUpWins, &nextWin);
2136
      rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pSup->numOfExprs, pOperatorInfo->pTaskInfo);
5
54liuyao 已提交
2137 2138
      taosArrayDestroy(pUpWins);
    }
2139 2140
    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx,
                                          numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
5
54liuyao 已提交
2141 2142 2143 2144 2145 2146 2147
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    pos->groupId = tableGroupId;
    pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
    *(int64_t*)pos->key = pResult->win.skey;
S
shenglian zhou 已提交
2148 2149
    forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
                                           TSDB_ORDER_ASC);
5
54liuyao 已提交
2150
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) {
5
54liuyao 已提交
2151
      saveResultRow(pResult, tableGroupId, pUpdated);
5
54liuyao 已提交
2152
    }
5
54liuyao 已提交
2153
    // window start(end) key interpolation
2154
    // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pSup->pCtx, pResult, &nextWin, startPos,
S
shenglian zhou 已提交
2155
    // forwardRows);
5
54liuyao 已提交
2156
    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
2157
    doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
X
Xiaoyu Wang 已提交
2158
                     tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
2159
    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
2160
    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
5
54liuyao 已提交
2161 2162 2163 2164 2165 2166 2167
    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
    if (startPos < 0) {
      break;
    }
  }
}

5
54liuyao 已提交
2168 2169 2170 2171
static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) {
  taosHashClear(pInfo->aggSup.pResultRowHashTable);
  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
  cleanupResultRowInfo(&pInfo->binfo.resultRowInfo);
2172
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
2173 2174 2175 2176 2177 2178 2179 2180 2181
}

static void clearUpdateDataBlock(SSDataBlock* pBlock) {
  if (pBlock->info.rows <= 0) {
    return;
  }
  blockDataCleanup(pBlock);
}

2182
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
5
54liuyao 已提交
2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193
  ASSERT(pDest->info.capacity >= pSource->info.rows);
  clearUpdateDataBlock(pDest);
  SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
  SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
  // copy timestamp column
  colDataAssign(pDestCol, pSourceCol, pSource->info.rows);
  for (int32_t i = 1; i < pDest->info.numOfCols; i++) {
    SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
    colDataAppendNNULL(pCol, 0, pSource->info.rows);
  }
  pDest->info.rows = pSource->info.rows;
2194 2195
  pDest->info.groupId = pSource->info.groupId;
  pDest->info.type = pSource->info.type;
5
54liuyao 已提交
2196 2197 2198 2199
  blockDataUpdateTsWindow(pDest, 0);
}

static int32_t getChildIndex(SSDataBlock* pBlock) {
2200
  return pBlock->info.childId;
5
54liuyao 已提交
2201 2202
}

5
54liuyao 已提交
2203 2204
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
  SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2205
  SOperatorInfo*                    downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
2206
  SArray*                           pUpdated = taosArrayInit(4, POINTER_BYTES);
5
54liuyao 已提交
2207
  TSKEY                             maxTs = INT64_MIN;
5
54liuyao 已提交
2208

2209 2210
  SExprSupp* pSup = &pOperator->exprSupp;

5
54liuyao 已提交
2211 2212 2213 2214
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  } else if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
5
54liuyao 已提交
2215
    if (pInfo->binfo.pRes->info.rows == 0) {
5
54liuyao 已提交
2216
      pOperator->status = OP_EXEC_DONE;
5
54liuyao 已提交
2217 2218 2219 2220 2221 2222 2223 2224 2225 2226
      if (isFinalInterval(pInfo) || pInfo->pUpdateRes->info.rows == 0) {
        if (!isFinalInterval(pInfo)) {
          // semi interval operator clear disk buffer
          clearStreamIntervalOperator(pInfo);
        }
        return NULL;
      }
      // process the rest of the data
      pOperator->status = OP_OPENED;
      return pInfo->pUpdateRes;
5
54liuyao 已提交
2227
    }
5
54liuyao 已提交
2228
    return pInfo->binfo.pRes;
5
54liuyao 已提交
2229 2230 2231 2232 2233
  }

  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
5
54liuyao 已提交
2234
      clearUpdateDataBlock(pInfo->pUpdateRes);
5
54liuyao 已提交
2235 2236
      break;
    }
2237

5
54liuyao 已提交
2238
    if (pBlock->info.type == STREAM_REPROCESS) {
X
Xiaoyu Wang 已提交
2239
      SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
2240
      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs,
X
Xiaoyu Wang 已提交
2241
                     pBlock, pUpWins);
2242
      if (isFinalInterval(pInfo)) {
5
54liuyao 已提交
2243
        int32_t                   childIndex = getChildIndex(pBlock);
X
Xiaoyu Wang 已提交
2244
        SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
2245
        SIntervalAggOperatorInfo* pChildInfo = pChildOp->info;
2246 2247 2248 2249 2250
        SExprSupp* pChildSup = &pChildOp->exprSupp;

        doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex,
                       pChildSup->numOfExprs, pBlock, NULL);
        rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs,
S
shenglian zhou 已提交
2251
                              pOperator->pTaskInfo);
5
54liuyao 已提交
2252 2253
        taosArrayDestroy(pUpWins);
        continue;
2254
      }
5
54liuyao 已提交
2255 2256
      removeResults(pUpWins, pUpdated);
      copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
2257
      taosArrayDestroy(pUpWins);
5
54liuyao 已提交
2258
      break;
2259
    } else if (pBlock->info.type == STREAM_GET_ALL && isFinalInterval(pInfo)) {
5
54liuyao 已提交
2260 2261
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
      continue;
5
54liuyao 已提交
2262
    }
5
54liuyao 已提交
2263

2264
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
5
54liuyao 已提交
2265
    doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated);
2266
    if (isFinalInterval(pInfo)) {
S
shenglian zhou 已提交
2267
      int32_t chIndex = getChildIndex(pBlock);
5
54liuyao 已提交
2268 2269 2270 2271 2272 2273 2274 2275 2276
      int32_t size = taosArrayGetSize(pInfo->pChildren);
      // if chIndex + 1 - size > 0, add new child
      for (int32_t i = 0; i < chIndex + 1 - size; i++) {
        SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0);
        if (!pChildOp) {
          longjmp(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
        }
        taosArrayPush(pInfo->pChildren, &pChildOp);
      }
S
shenglian zhou 已提交
2277
      SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
5
54liuyao 已提交
2278
      SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
2279
      setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
5
54liuyao 已提交
2280 2281
      doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL);
    }
5
54liuyao 已提交
2282
    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
2283
  }
S
shenglian zhou 已提交
2284

5
54liuyao 已提交
2285
  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
5
54liuyao 已提交
2286
  if (isFinalInterval(pInfo)) {
5
54liuyao 已提交
2287
    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
5
54liuyao 已提交
2288 2289
  }

2290
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
5
54liuyao 已提交
2291 2292 2293 2294
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
  pOperator->status = OP_RES_TO_RETURN;
5
54liuyao 已提交
2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306
  if (pInfo->binfo.pRes->info.rows == 0) {
    pOperator->status = OP_EXEC_DONE;
    if (pInfo->pUpdateRes->info.rows == 0) {
      return NULL;
    }
    // process the rest of the data
    pOperator->status = OP_OPENED;
    return pInfo->pUpdateRes;
  }
  return pInfo->binfo.pRes;
}

S
shenglian zhou 已提交
2307 2308 2309
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                     SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
  SIntervalPhysiNode*               pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
5
54liuyao 已提交
2310
  SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo));
S
shenglian zhou 已提交
2311
  SOperatorInfo*                    pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
2312 2313 2314
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
2315

5
54liuyao 已提交
2316
  pInfo->order = TSDB_ORDER_ASC;
S
shenglian zhou 已提交
2317 2318 2319 2320 2321 2322 2323 2324
  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                .sliding = pIntervalPhyNode->sliding,
                                .intervalUnit = pIntervalPhyNode->intervalUnit,
                                .slidingUnit = pIntervalPhyNode->slidingUnit,
                                .offset = pIntervalPhyNode->offset,
                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pIntervalPhyNode->window.watermark,
5
54liuyao 已提交
2325 2326
      .calTrigger = pIntervalPhyNode->window.triggerType,
      .maxTs = INT64_MIN,
S
shenglian zhou 已提交
2327
  };
2328
  ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY);
5
54liuyao 已提交
2329 2330 2331
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);
S
shenglian zhou 已提交
2332 2333
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
5
54liuyao 已提交
2334
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
2335 2336 2337 2338

  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);

2339
  ASSERT(numOfCols > 0);
2340
  increaseTs(pOperator->exprSupp.pCtx);
5
54liuyao 已提交
2341 2342 2343 2344
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
2345
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361
  pInfo->pChildren = NULL;
  if (numOfChild > 0) {
    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(SOperatorInfo));
    for (int32_t i = 0; i < numOfChild; i++) {
      SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0);
      if (pChildOp) {
        taosArrayPush(pInfo->pChildren, &pChildOp);
        continue;
      }
      goto _error;
    }
  }
  // semi interval operator does not catch result
  if (!isFinalInterval(pInfo)) {
    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
  }
S
shenglian zhou 已提交
2362
  pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
5
54liuyao 已提交
2363 2364
  pInfo->pUpdateRes->info.type = STREAM_REPROCESS;
  blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
2365
  pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
5
54liuyao 已提交
2366 2367 2368 2369 2370

  pOperator->name = "StreamFinalIntervalOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
2371
  pOperator->exprSupp.pExprInfo = pExprInfo;
5
54liuyao 已提交
2372
  pOperator->pTaskInfo = pTaskInfo;
2373
  pOperator->exprSupp.numOfExprs = numOfCols;
5
54liuyao 已提交
2374 2375
  pOperator->info = pInfo;

S
shenglian zhou 已提交
2376 2377 2378
  pOperator->fpSet =
      createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
  destroyStreamFinalIntervalOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
5
54liuyao 已提交
2393
}
5
54liuyao 已提交
2394 2395 2396

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
  taosMemoryFreeClear(pSup->pKeyBuf);
2397 2398 2399 2400 2401 2402
  void **pIte = NULL;
  while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) {
    SArray *pWins = (SArray *) (*pIte);
    taosArrayDestroy(pWins);
  }
  taosHashCleanup(pSup->pResultRows);
5
54liuyao 已提交
2403 2404 2405 2406 2407
  destroyDiskbasedBuf(pSup->pResultBuf);
}

void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;
2408
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
2409 2410
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
2411 2412 2413
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
2414
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
2415 2416 2417 2418 2419 2420
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
      destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput);
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
5
54liuyao 已提交
2421 2422
}

2423 2424 2425 2426
int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock) {
  initExprSupp(pSup, pExprInfo, numOfCols);
  initBasicInfo(pBasicInfo, pResultBlock);
  
5
54liuyao 已提交
2427
  for (int32_t i = 0; i < numOfCols; ++i) {
2428
    pSup->pCtx[i].pBuf = NULL;
5
54liuyao 已提交
2429
  }
2430
  
2431
  ASSERT(numOfCols > 0);
2432
  increaseTs(pSup->pCtx);
5
54liuyao 已提交
2433 2434 2435 2436 2437 2438 2439 2440
  return TSDB_CODE_SUCCESS;
}

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {
  for (int i = 0; i < nums; i++) {
    pDummy[i].functionId = pCtx[i].functionId;
  }
}
X
Xiaoyu Wang 已提交
2441 2442
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
                    uint8_t type) {
5
54liuyao 已提交
2443 2444
  ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
  SStreamBlockScanInfo* pScanInfo = downstream->info;
X
Xiaoyu Wang 已提交
2445
  pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
5
54liuyao 已提交
2446
  pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
5
54liuyao 已提交
2447 2448
}

2449 2450 2451 2452
int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo));
}

2453 2454 2455 2456 2457 2458 2459
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
  SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
  int32_t numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
  int32_t code = TSDB_CODE_OUT_OF_MEMORY;
X
Xiaoyu Wang 已提交
2460
  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
2461
  SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
2462 2463 2464 2465 2466
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

  initResultSizeInfo(pOperator, 4096);
2467
  SExprSupp* pSup = &pOperator->exprSupp;
5
54liuyao 已提交
2468

2469
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
2470 2471 2472
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
2473
  
2474
  
2475
  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
2476 2477 2478
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
X
Xiaoyu Wang 已提交
2479

5
54liuyao 已提交
2480 2481 2482 2483
  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }
2484
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
2485

H
Haojun Liao 已提交
2486 2487
  pInfo->twAggSup = (STimeWindowAggSupp) {
      .waterMark = pSessionNode->window.watermark,
2488 2489
      .calTrigger = pSessionNode->window.triggerType,
      .maxTs = INT64_MIN};
H
Haojun Liao 已提交
2490 2491

  initResultRowInfo(&pInfo->binfo.resultRowInfo);
5
54liuyao 已提交
2492 2493 2494
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

  pInfo->primaryTsIndex = tsSlotId;
2495
  pInfo->gap = pSessionNode->gap;
5
54liuyao 已提交
2496 2497 2498 2499 2500 2501 2502
  pInfo->binfo.pRes = pResBlock;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
  pInfo->pDelRes = createOneDataBlock(pResBlock, false);
  blockDataEnsureCapacity(pInfo->pDelRes, 64);
2503
  pInfo->pChildren = NULL;
5
54liuyao 已提交
2504 2505

  pOperator->name = "StreamSessionWindowAggOperator";
2506
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
5
54liuyao 已提交
2507 2508
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
2509 2510
  pOperator->exprSupp.pExprInfo = pExprInfo;
  pOperator->exprSupp.numOfExprs = numOfCols;
5
54liuyao 已提交
2511
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
2512 2513 2514
  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
                          aggEncodeResultRow, aggDecodeResultRow, NULL);
5
54liuyao 已提交
2515
  pOperator->pTaskInfo = pTaskInfo;
X
Xiaoyu Wang 已提交
2516
  initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType);
5
54liuyao 已提交
2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531
  code = appendDownstream(pOperator, &downstream, 1);
  return pOperator;

_error:
  if (pInfo != NULL) {
    destroyStreamSessionAggOperatorInfo(pInfo, numOfCols);
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}

int64_t getSessionWindowEndkey(void* data, int32_t index) {
X
Xiaoyu Wang 已提交
2532
  SArray*            pWinInfos = (SArray*)data;
5
54liuyao 已提交
2533 2534 2535 2536 2537 2538
  SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index);
  return pWin->win.ekey;
}
static bool isInWindow(SResultWindowInfo* pWin, TSKEY ts, int64_t gap) {
  int64_t sGap = ts - pWin->win.skey;
  int64_t eGap = pWin->win.ekey - ts;
X
Xiaoyu Wang 已提交
2539
  if ((sGap < 0 && sGap >= -gap) || (eGap < 0 && eGap >= -gap) || (sGap >= 0 && eGap >= 0)) {
5
54liuyao 已提交
2540 2541 2542 2543 2544
    return true;
  }
  return false;
}

X
Xiaoyu Wang 已提交
2545 2546
static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) {
  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
5
54liuyao 已提交
2547 2548 2549 2550
  return taosArrayInsert(pWinInfos, index, &win);
}

static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) {
X
Xiaoyu Wang 已提交
2551
  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
5
54liuyao 已提交
2552 2553 2554
  return taosArrayPush(pWinInfos, &win);
}

2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570
SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) {
  void** ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t));
  SArray* pWinInfos = NULL;
  if (ite == NULL) {
    pWinInfos = taosArrayInit(1024, pAggSup->valueSize);
    taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void *));
  } else {
    pWinInfos = *ite;
  }
  return pWinInfos;
}

SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int64_t gap, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

5
54liuyao 已提交
2571 2572
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
5
54liuyao 已提交
2573
    *pIndex = 0;
5
54liuyao 已提交
2574 2575 2576
    return addNewSessionWindow(pWinInfos, ts);
  }
  // find the first position which is smaller than the key
X
Xiaoyu Wang 已提交
2577
  int32_t            index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getSessionWindowEndkey);
5
54liuyao 已提交
2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isInWindow(pWin, ts, gap)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isInWindow(pWin, ts, gap)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewSessionWindow(pWinInfos, ts);
  }
5
54liuyao 已提交
2599 2600
  *pIndex = index + 1;
  return insertNewSessionWindow(pWinInfos, ts, index + 1);
5
54liuyao 已提交
2601 2602
}

X
Xiaoyu Wang 已提交
2603 2604
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, int32_t start, int64_t gap,
                                SHashObj* pStDeleted) {
5
54liuyao 已提交
2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620
  for (int32_t i = start; i < rows; ++i) {
    if (!isInWindow(pWinInfo, pTs[i], gap)) {
      return i - start;
    }
    if (pWinInfo->win.skey > pTs[i]) {
      if (pStDeleted && pWinInfo->isOutput) {
        taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY));
        pWinInfo->isOutput = false;
      }
      pWinInfo->win.skey = pTs[i];
    }
    pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pTs[i]);
  }
  return rows - start;
}

X
Xiaoyu Wang 已提交
2621
static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,
2622
                                  uint64_t groupId, int32_t numOfOutput, int32_t* rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
2623
                                  SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) {
5
54liuyao 已提交
2624 2625
  assert(pWinInfo->win.skey <= pWinInfo->win.ekey);
  // too many time window in query
2626
  int32_t size = taosArrayGetSize(pAggSup->pCurWins);
5
54liuyao 已提交
2627 2628 2629
  if (size > MAX_INTERVAL_TIME_WINDOW) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
  }
X
Xiaoyu Wang 已提交
2630

5
54liuyao 已提交
2631
  if (pWinInfo->pos.pageId == -1) {
2632
    *pResult = getNewResultRow(pAggSup->pResultBuf, groupId, pAggSup->resultRowSize);
5
54liuyao 已提交
2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650
    if (*pResult == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }
    initResultRow(*pResult);

    // add a new result set for a new group
    pWinInfo->pos.pageId = (*pResult)->pageId;
    pWinInfo->pos.offset = (*pResult)->offset;
  } else {
    *pResult = getResultRowByPos(pAggSup->pResultBuf, &pWinInfo->pos);
    if (!(*pResult)) {
      qError("getResultRowByPos return NULL, TID:%s", GET_TASKID(pTaskInfo));
      return TSDB_CODE_FAILED;
    }
  }

  // set time window for current result
  (*pResult)->win = pWinInfo->win;
2651
  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
5
54liuyao 已提交
2652 2653 2654
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
2655 2656 2657
static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup,
                                  SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock,
                                  SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
2658 2659 2660 2661
                                  int32_t numOutput, SOperatorInfo* pOperator) {
  SExprSupp* pSup = &pOperator->exprSupp;
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

X
Xiaoyu Wang 已提交
2662 2663
  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId);
  TSKEY*           tsCols = (int64_t*)pColDataInfo->pData;
2664 2665
  int32_t          code = setWindowOutputBuf(pCurWin, pResult, pSup->pCtx, pSDataBlock->info.groupId, numOutput,
                                             pSup->rowEntryInfoOffset, pAggSup, pTaskInfo);
5
54liuyao 已提交
2666 2667 2668
  if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }
5
54liuyao 已提交
2669
  updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, true);
2670
  doApplyFunctions(pTaskInfo, pSup->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols,
X
Xiaoyu Wang 已提交
2671
                   pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC);
5
54liuyao 已提交
2672 2673 2674
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
2675 2676
static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                              SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows,
2677
                              int32_t numOutput, SOperatorInfo * pOperator) {
X
Xiaoyu Wang 已提交
2678
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
2679
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
2680 2681
}

X
Xiaoyu Wang 已提交
2682 2683
static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock,
                                   SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex,
2684
                                   int32_t winRows, int32_t numOutput, SOperatorInfo * pOperator) {
X
Xiaoyu Wang 已提交
2685
  return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData,
2686
                            pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator);
5
54liuyao 已提交
2687 2688
}

5
54liuyao 已提交
2689 2690
int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) {
  SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex);
X
Xiaoyu Wang 已提交
2691
  int32_t            size = taosArrayGetSize(pWinInfos);
5
54liuyao 已提交
2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < size; i++) {
    SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i);
    if (!isInWindow(pCurWin, pWinInfo->win.skey, gap)) {
      return i - startIndex - 1;
    }
  }

  return size - startIndex - 1;
}

5
54liuyao 已提交
2703
void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId,
2704 2705 2706 2707
                       int32_t numOfOutput, SHashObj* pStUpdated, SHashObj* pStDeleted, SOperatorInfo* pOperator) {
  SExprSupp* pSup = &pOperator->exprSupp;
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

2708
  SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex);
X
Xiaoyu Wang 已提交
2709
  SResultRow*        pCurResult = NULL;
2710
  setWindowOutputBuf(pCurWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
2711
                     &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
2712
  num += startIndex + 1;
2713
  ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins));
5
54liuyao 已提交
2714 2715
  // Just look for the window behind StartIndex
  for (int32_t i = startIndex + 1; i < num; i++) {
2716
    SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i);
X
Xiaoyu Wang 已提交
2717
    SResultRow*        pWinResult = NULL;
2718
    setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
2719
                       &pInfo->streamAggSup, pTaskInfo);
5
54liuyao 已提交
2720
    pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey);
2721
    compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo);
5
54liuyao 已提交
2722 2723 2724 2725 2726
    taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition));
    if (pWinInfo->isOutput) {
      taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY));
      pWinInfo->isOutput = false;
    }
2727
    taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
5
54liuyao 已提交
2728 2729 2730
  }
}

5
54liuyao 已提交
2731 2732 2733 2734 2735
typedef struct SWinRes {
  TSKEY    ts;
  uint64_t groupId;
} SWinRes;

X
Xiaoyu Wang 已提交
2736 2737 2738
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
                                   SHashObj* pStDeleted) {
  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
2739
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2740
  bool                           masterScan = true;
2741
  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
5
54liuyao 已提交
2742
  uint64_t                       groupId = pSDataBlock->info.groupId;
X
Xiaoyu Wang 已提交
2743 2744 2745 2746 2747 2748
  int64_t                        gap = pInfo->gap;
  int64_t                        code = TSDB_CODE_SUCCESS;

  int32_t     step = 1;
  bool        ascScan = true;
  TSKEY*      tsCols = NULL;
5
54liuyao 已提交
2749
  SResultRow* pResult = NULL;
X
Xiaoyu Wang 已提交
2750
  int32_t     winRows = 0;
5
54liuyao 已提交
2751 2752

  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
2753
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
5
54liuyao 已提交
2754 2755
    tsCols = (int64_t*)pColDataInfo->pData;
  } else {
X
Xiaoyu Wang 已提交
2756
    return;
5
54liuyao 已提交
2757
  }
X
Xiaoyu Wang 已提交
2758

5
54liuyao 已提交
2759
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
2760 2761
  for (int32_t i = 0; i < pSDataBlock->info.rows;) {
    int32_t            winIndex = 0;
5
54liuyao 已提交
2762
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], groupId, gap, &winIndex);
X
Xiaoyu Wang 已提交
2763
    winRows = updateSessionWindowInfo(pCurWin, tsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted);
2764
    code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
2765 2766 2767 2768
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    // window start(end) key interpolation
2769
    // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pSup->pCtx, pResult, &nextWin, startPos,
X
Xiaoyu Wang 已提交
2770
    // forwardRows,
5
54liuyao 已提交
2771
    //                             pInfo->order, false);
2772
    int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap);
5
54liuyao 已提交
2773
    if (winNum > 0) {
2774
      compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, pStDeleted, pOperator);
5
54liuyao 已提交
2775
    }
5
54liuyao 已提交
2776 2777
    pCurWin->isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
2778 2779
      SWinRes value = {.ts = pCurWin->win.skey, .groupId = groupId};
      code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinRes));
5
54liuyao 已提交
2780 2781 2782 2783
      if (code != TSDB_CODE_SUCCESS) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
      pCurWin->isOutput = true;
5
54liuyao 已提交
2784 2785 2786 2787 2788
    }
    i += winRows;
  }
}

2789
static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock,
X
Xiaoyu Wang 已提交
2790
                                  int32_t tsIndex, int32_t numOfOutput, int64_t gap, SArray* result) {
5
54liuyao 已提交
2791
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
X
Xiaoyu Wang 已提交
2792 2793
  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
  int32_t          step = 0;
5
54liuyao 已提交
2794
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
2795
    int32_t            winIndex = 0;
2796
    SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], pBlock->info.groupId, gap, &winIndex);
5
54liuyao 已提交
2797
    step = updateSessionWindowInfo(pCurWin, tsCols, pBlock->info.rows, i, gap, NULL);
2798
    ASSERT(isInWindow(pCurWin, tsCols[i], gap));
2799
    doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
2800 2801 2802
    if (result) {
      taosArrayPush(result, pCurWin);
    }
5
54liuyao 已提交
2803 2804 2805
  }
}

5
54liuyao 已提交
2806
static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) {
X
Xiaoyu Wang 已提交
2807
  void*  pData = NULL;
5
54liuyao 已提交
2808
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
2809
  while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) {
5
54liuyao 已提交
2810 2811 2812 2813 2814 2815
    void* key = taosHashGetKey(pData, &keyLen);
    ASSERT(keyLen == sizeof(SResultRowPosition));
    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
    if (pos == NULL) {
      return TSDB_CODE_QRY_OUT_OF_MEMORY;
    }
5
54liuyao 已提交
2816
    pos->groupId = ((SWinRes*)pData)->groupId;
5
54liuyao 已提交
2817
    pos->pos = *(SResultRowPosition*)key;
5
54liuyao 已提交
2818
    *(int64_t*)pos->key = ((SWinRes*)pData)->ts;
5
54liuyao 已提交
2819 2820 2821 2822 2823 2824 2825 2826
    taosArrayPush(pUpdated, &pos);
  }
  return TSDB_CODE_SUCCESS;
}

void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
  blockDataCleanup(pBlock);
  size_t keyLen = 0;
X
Xiaoyu Wang 已提交
2827
  while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
5
54liuyao 已提交
2828
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
X
Xiaoyu Wang 已提交
2829
    colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false);
5
54liuyao 已提交
2830 2831
    for (int32_t i = 1; i < pBlock->info.numOfCols; i++) {
      pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
X
Xiaoyu Wang 已提交
2832
      colDataAppendNULL(pColInfoData, pBlock->info.rows);
5
54liuyao 已提交
2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843
    }
    pBlock->info.rows += 1;
    if (pBlock->info.rows + 1 >= pBlock->info.capacity) {
      break;
    }
  }
  if ((*Ite) == NULL) {
    taosHashClear(pStDeleted);
  }
}

X
Xiaoyu Wang 已提交
2844
static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId,
2845 2846 2847 2848
                              int32_t numOfOutput, SOperatorInfo* pOperator) {
  SExprSupp* pSup = &pOperator->exprSupp;
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

2849 2850
  int32_t size = taosArrayGetSize(pWinArray);
  ASSERT(pInfo->pChildren);
2851

2852 2853
  for (int32_t i = 0; i < size; i++) {
    SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i);
X
Xiaoyu Wang 已提交
2854
    SResultRow*        pCurResult = NULL;
2855
    setWindowOutputBuf(pParentWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
X
Xiaoyu Wang 已提交
2856
                       &pInfo->streamAggSup, pTaskInfo);
2857 2858
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
    for (int32_t j = 0; j < numOfChildren; j++) {
X
Xiaoyu Wang 已提交
2859
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
2860
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
2861
      SArray*                        pChWins = getWinInfos(&pChInfo->streamAggSup, groupId);
X
Xiaoyu Wang 已提交
2862 2863
      int32_t                        chWinSize = taosArrayGetSize(pChWins);
      int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey);
2864 2865 2866 2867
      for (int32_t k = index; k > 0 && k < chWinSize; k++) {
        SResultWindowInfo* pcw = taosArrayGet(pChWins, k);
        if (pParentWin->win.skey <= pcw->win.skey && pcw->win.ekey <= pParentWin->win.ekey) {
          SResultRow* pChResult = NULL;
2868 2869 2870
          setWindowOutputBuf(pcw, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
                             pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
          compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo);
2871 2872 2873 2874 2875 2876 2877 2878
          continue;
        }
        break;
      }
    }
  }
}

X
Xiaoyu Wang 已提交
2879
bool isFinalSession(SStreamSessionAggOperatorInfo* pInfo) { return pInfo->pChildren != NULL; }
2880

X
Xiaoyu Wang 已提交
2881 2882 2883
typedef SResultWindowInfo* (*__get_win_info_)(void*);
SResultWindowInfo* getSessionWinInfo(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getStateWinInfo(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
5
54liuyao 已提交
2884

2885
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed,
5
54liuyao 已提交
2886
    __get_win_info_ fn) {
5
54liuyao 已提交
2887
  // Todo(liuyao) save window to tdb
2888
  void **pIte = NULL;
5
54liuyao 已提交
2889
  size_t keyLen = 0;
2890
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
5
54liuyao 已提交
2891
    uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen);
2892 2893 2894 2895 2896 2897 2898 2899 2900
    SArray *pWins = (SArray *) (*pIte);
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) {
        if (!pSeWin->isClosed) {
          pSeWin->isClosed = true;
          if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
5
54liuyao 已提交
2901
            int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed);
2902 2903
            pSeWin->isOutput = true;
          }
5
54liuyao 已提交
2904
        }
2905
        continue;
5
54liuyao 已提交
2906
      }
2907
      break;
5
54liuyao 已提交
2908 2909 2910 2911 2912
    }
  }
  return TSDB_CODE_SUCCESS;
}

2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924
int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) {
  void **pIte = NULL;
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
    SArray *pWins = (SArray *) (*pIte);
    int32_t size = taosArrayGetSize(pWins);
    for (int32_t i = 0; i < size; i++) {
      void*              pWin = taosArrayGet(pWins, i);
      SResultWindowInfo* pSeWin = fn(pWin);
      if (!pSeWin->isClosed) {
        int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed);
        pSeWin->isOutput = true;
      }
5
54liuyao 已提交
2925 2926 2927 2928 2929
    }
  }
  return TSDB_CODE_SUCCESS;
}

5
54liuyao 已提交
2930
static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
5
54liuyao 已提交
2931 2932 2933 2934
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

2935
  SExprSupp* pSup = &pOperator->exprSupp;
5
54liuyao 已提交
2936
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
2937
  SOptrBasicInfo*                pBInfo = &pInfo->binfo;
5
54liuyao 已提交
2938 2939 2940 2941 2942
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
2943
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
2944
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
5
54liuyao 已提交
2945 2946 2947 2948 2949
      doSetOperatorCompleted(pOperator);
    }
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
2950 2951
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
2952
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
2953
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
2954 2955 2956 2957 2958
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
2959

X
Xiaoyu Wang 已提交
2960 2961
    if (pBlock->info.type == STREAM_REPROCESS) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
2962
      doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, pInfo->gap, pWins);
2963
      if (isFinalSession(pInfo)) {
X
Xiaoyu Wang 已提交
2964 2965
        int32_t                        childIndex = 0;  // Todo(liuyao) get child id from SSDataBlock
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
2966
        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
2967
        doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, 0, pChildOp->exprSupp.numOfExprs,
X
Xiaoyu Wang 已提交
2968
                              pChildInfo->gap, NULL);
2969
        rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
2970 2971
      }
      taosArrayDestroy(pWins);
5
54liuyao 已提交
2972
      continue;
2973
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
2974 2975
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getSessionWinInfo);
      continue;
5
54liuyao 已提交
2976
    }
5
54liuyao 已提交
2977

2978
    // the pDataBlock are always the same one, no need to call this again
2979
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
2980
    if (isFinalSession(pInfo)) {
X
Xiaoyu Wang 已提交
2981
      int32_t         childIndex = 0;  // Todo(liuyao) get child id from SSDataBlock
2982
      SOptrBasicInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
5
54liuyao 已提交
2983
      doStreamSessionAggImpl(pOperator, pBlock, NULL, NULL);
2984
    }
5
54liuyao 已提交
2985
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted);
5
54liuyao 已提交
2986
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
5
54liuyao 已提交
2987 2988 2989
  }
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
H
Haojun Liao 已提交
2990

5
54liuyao 已提交
2991
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
X
Xiaoyu Wang 已提交
2992
                     getSessionWinInfo);
5
54liuyao 已提交
2993
  copyUpdateResult(pStUpdated, pUpdated);
5
54liuyao 已提交
2994
  taosHashCleanup(pStUpdated);
5
54liuyao 已提交
2995

2996 2997
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
2998 2999 3000 3001 3002 3003
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
3004
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
3005 3006
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}
3007

3008 3009 3010 3011
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream,
    SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) {
  int32_t        code = TSDB_CODE_OUT_OF_MEMORY;
  SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo);
3012 3013 3014 3015
  if (pOperator == NULL) {
    goto _error;
  }
  pOperator->name = "StreamFinalSessionWindowAggOperator";
3016
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION;
3017
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3018
  pInfo->pChildren = taosArrayInit(8, sizeof(void*));
3019
  for (int32_t i = 0; i < numOfChild; i++) {
X
Xiaoyu Wang 已提交
3020
    SOperatorInfo* pChild =
3021
        createStreamSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo);
3022 3023 3024 3025 3026 3027 3028 3029 3030
    if (pChild == NULL) {
      goto _error;
    }
    taosArrayPush(pInfo->pChildren, &pChild);
  }
  return pOperator;

_error:
  if (pInfo != NULL) {
3031
    destroyStreamSessionAggOperatorInfo(pInfo, pOperator->exprSupp.numOfExprs);
3032 3033 3034 3035 3036 3037 3038
  }

  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
5
54liuyao 已提交
3039 3040

void destroyStreamStateOperatorInfo(void* param, int32_t numOfOutput) {
X
Xiaoyu Wang 已提交
3041
  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;
3042
  cleanupBasicInfo(&pInfo->binfo);
5
54liuyao 已提交
3043 3044 3045 3046 3047
  destroyStreamAggSupporter(&pInfo->streamAggSup);
  cleanupGroupResInfo(&pInfo->groupResInfo);
  if (pInfo->pChildren != NULL) {
    int32_t size = taosArrayGetSize(pInfo->pChildren);
    for (int32_t i = 0; i < size; i++) {
X
Xiaoyu Wang 已提交
3048
      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, i);
5
54liuyao 已提交
3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061
      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
      destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput);
      taosMemoryFreeClear(pChild);
      taosMemoryFreeClear(pChInfo);
    }
  }
}

int64_t getStateWinTsKey(void* data, int32_t index) {
  SStateWindowInfo* pStateWin = taosArrayGet(data, index);
  return pStateWin->winInfo.win.ekey;
}

X
Xiaoyu Wang 已提交
3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073
SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
3074 3075 3076 3077 3078 3079 3080 3081
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayPush(pWinInfos, &win);
}

X
Xiaoyu Wang 已提交
3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093
SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) {
  SStateWindowInfo win = {
      .stateKey.bytes = pCol->bytes,
      .stateKey.type = pCol->type,
      .stateKey.pData = taosMemoryCalloc(1, pCol->bytes),
      .winInfo.pos.offset = -1,
      .winInfo.pos.pageId = -1,
      .winInfo.win.skey = ts,
      .winInfo.win.ekey = ts,
      .winInfo.isOutput = false,
      .winInfo.isClosed = false,
  };
5
54liuyao 已提交
3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112
  if (IS_VAR_DATA_TYPE(win.stateKey.type)) {
    varDataCopy(win.stateKey.pData, pKeyData);
  } else {
    memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes);
  }
  return taosArrayInsert(pWinInfos, index, &win);
}

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {
  if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) {
    return true;
  }
  return false;
}

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {
  return pKeyData && compareVal(pKeyData, &pWin->stateKey);
}

3113 3114 3115
SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
X
Xiaoyu Wang 已提交
3116 3117
  int32_t           size = taosArrayGetSize(pWinInfos);
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index + 1;
      return pWin;
    }
  }
  *pIndex = 0;
  return NULL;
}

3138 3139 3140 3141
SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts,
    uint64_t groupId, char* pKeyData, SColumn* pCol, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;
5
54liuyao 已提交
3142 3143 3144 3145 3146
  int32_t size = taosArrayGetSize(pWinInfos);
  if (size == 0) {
    *pIndex = 0;
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
X
Xiaoyu Wang 已提交
3147
  int32_t           index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey);
5
54liuyao 已提交
3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180
  SStateWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isTsInWindow(pWin, ts)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index + 1 < size) {
    pWin = taosArrayGet(pWinInfos, index + 1);
    if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index + 1;
      return pWin;
    }
  }

  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
    if (isEqualStateKey(pWin, pKeyData)) {
      *pIndex = index;
      return pWin;
    }
  }

  if (index == size - 1) {
    *pIndex = taosArrayGetSize(pWinInfos);
    return addNewStateWindow(pWinInfos, ts, pKeyData, pCol);
  }
  *pIndex = index + 1;
  return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol);
}

X
Xiaoyu Wang 已提交
3181 3182
int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, SColumnInfoData* pKeyCol, int32_t rows,
                              int32_t start, bool* allEqual, SHashObj* pSeDelete) {
5
54liuyao 已提交
3183 3184 3185 3186 3187
  *allEqual = true;
  SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex);
  for (int32_t i = start; i < rows; ++i) {
    char* pKeyData = colDataGetData(pKeyCol, i);
    if (!isTsInWindow(pWinInfo, pTs[i])) {
X
Xiaoyu Wang 已提交
3188
      if (isEqualStateKey(pWinInfo, pKeyData)) {
5
54liuyao 已提交
3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202
        int32_t size = taosArrayGetSize(pWinInfos);
        if (winIndex + 1 < size) {
          SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1);
          // ts belongs to the next window
          if (pTs[i] >= pNextWin->winInfo.win.skey) {
            return i - start;
          }
        }
      } else {
        return i - start;
      }
    }
    if (pWinInfo->winInfo.win.skey > pTs[i]) {
      if (pSeDelete && pWinInfo->winInfo.isOutput) {
X
Xiaoyu Wang 已提交
3203 3204
        taosHashPut(pSeDelete, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &pWinInfo->winInfo.win.skey,
                    sizeof(TSKEY));
5
54liuyao 已提交
3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217
        pWinInfo->winInfo.isOutput = false;
      }
      pWinInfo->winInfo.win.skey = pTs[i];
    }
    pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]);
    if (!isEqualStateKey(pWinInfo, pKeyData)) {
      *allEqual = false;
    }
  }
  return rows - start;
}

void deleteWindow(SArray* pWinInfos, int32_t index) {
X
Xiaoyu Wang 已提交
3218
  ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos));
5
54liuyao 已提交
3219 3220 3221
  taosArrayRemove(pWinInfos, index);
}

X
Xiaoyu Wang 已提交
3222 3223
static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int32_t tsIndex, SColumn* pCol,
                                int32_t keyIndex, SHashObj* pSeUpdated, SHashObj* pSeDeleted) {
5
54liuyao 已提交
3224 3225
  SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
  SColumnInfoData* pKeyColInfo = taosArrayGet(pBlock->pDataBlock, keyIndex);
X
Xiaoyu Wang 已提交
3226 3227 3228
  TSKEY*           tsCol = (TSKEY*)pTsColInfo->pData;
  bool             allEqual = false;
  int32_t          step = 1;
5
54liuyao 已提交
3229
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
X
Xiaoyu Wang 已提交
3230 3231
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
3232
    SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], pBlock->info.groupId, &winIndex);
5
54liuyao 已提交
3233 3234 3235
    if (!pCurWin) {
      continue;
    }
3236
    step = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual,
X
Xiaoyu Wang 已提交
3237
                                 pSeDeleted);
5
54liuyao 已提交
3238 3239 3240
    ASSERT(isTsInWindow(pCurWin, tsCol[i]) || isEqualStateKey(pCurWin, pKeyData));
    taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win);
    taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
3241
    deleteWindow(pAggSup->pCurWins, winIndex);
5
54liuyao 已提交
3242 3243 3244
  }
}

X
Xiaoyu Wang 已提交
3245 3246 3247
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated,
                                 SHashObj* pStDeleted) {
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
5
54liuyao 已提交
3248
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3249
  bool                         masterScan = true;
3250
  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
X
Xiaoyu Wang 已提交
3251 3252 3253 3254 3255 3256 3257
  int64_t                      groupId = pSDataBlock->info.groupId;
  int64_t                      code = TSDB_CODE_SUCCESS;
  int32_t                      step = 1;
  bool                         ascScan = true;
  TSKEY*                       tsCols = NULL;
  SResultRow*                  pResult = NULL;
  int32_t                      winRows = 0;
5
54liuyao 已提交
3258
  if (pSDataBlock->pDataBlock != NULL) {
X
Xiaoyu Wang 已提交
3259 3260
    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
    tsCols = (int64_t*)pColDataInfo->pData;
5
54liuyao 已提交
3261
  } else {
X
Xiaoyu Wang 已提交
3262
    return;
5
54liuyao 已提交
3263
  }
X
Xiaoyu Wang 已提交
3264

5
54liuyao 已提交
3265
  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
X
Xiaoyu Wang 已提交
3266 3267 3268 3269 3270
  SColumnInfoData*     pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
  for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
    char*             pKeyData = colDataGetData(pKeyColInfo, i);
    int32_t           winIndex = 0;
    bool              allEqual = true;
3271 3272 3273 3274 3275
    SStateWindowInfo* pCurWin = 
        getStateWindow(pAggSup, tsCols[i], pSDataBlock->info.groupId, pKeyData,
            &pInfo->stateCol, &winIndex);
    winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, pKeyColInfo,
        pSDataBlock->info.rows, i, &allEqual, pInfo->pSeDeleted);
5
54liuyao 已提交
3276 3277 3278
    if (!allEqual) {
      taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win);
      taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
3279
      deleteWindow(pAggSup->pCurWins, winIndex);
5
54liuyao 已提交
3280 3281
      continue;
    }
3282
    code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pOperator);
5
54liuyao 已提交
3283 3284 3285 3286 3287
    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }
    pCurWin->winInfo.isClosed = false;
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
5
54liuyao 已提交
3288 3289 3290
      SWinRes value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId};
      code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition),
          &value, sizeof(SWinRes));
5
54liuyao 已提交
3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303
      if (code != TSDB_CODE_SUCCESS) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
      pCurWin->winInfo.isOutput = true;
    }
  }
}

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

3304
  SExprSupp* pSup = &pOperator->exprSupp;
5
54liuyao 已提交
3305
  SStreamStateAggOperatorInfo* pInfo = pOperator->info;
X
Xiaoyu Wang 已提交
3306
  SOptrBasicInfo*              pBInfo = &pInfo->binfo;
5
54liuyao 已提交
3307 3308 3309 3310 3311
  if (pOperator->status == OP_RES_TO_RETURN) {
    doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
    if (pInfo->pDelRes->info.rows > 0) {
      return pInfo->pDelRes;
    }
X
Xiaoyu Wang 已提交
3312
    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
3313
    if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
5
54liuyao 已提交
3314 3315 3316 3317 3318
      doSetOperatorCompleted(pOperator);
    }
    return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
  }

X
Xiaoyu Wang 已提交
3319 3320
  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  SHashObj*      pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
5
54liuyao 已提交
3321
  SOperatorInfo* downstream = pOperator->pDownstream[0];
5
54liuyao 已提交
3322
  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
5
54liuyao 已提交
3323 3324 3325 3326 3327
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
      break;
    }
3328

5
54liuyao 已提交
3329
    if (pBlock->info.type == STREAM_REPROCESS) {
X
Xiaoyu Wang 已提交
3330 3331
      doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
                          pSeUpdated, pInfo->pSeDeleted);
5
54liuyao 已提交
3332
      continue;
3333
    } else if (pBlock->info.type == STREAM_GET_ALL) {
5
54liuyao 已提交
3334 3335
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getStateWinInfo);
      continue;
5
54liuyao 已提交
3336
    }
3337 3338

    // the pDataBlock are always the same one, no need to call this again
3339
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
5
54liuyao 已提交
3340 3341 3342 3343 3344
    doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
  }
  // restore the value
  pOperator->status = OP_RES_TO_RETURN;
X
Xiaoyu Wang 已提交
3345

5
54liuyao 已提交
3346
  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
X
Xiaoyu Wang 已提交
3347
                     getStateWinInfo);
5
54liuyao 已提交
3348
  copyUpdateResult(pSeUpdated, pUpdated);
5
54liuyao 已提交
3349 3350
  taosHashCleanup(pSeUpdated);

3351 3352
  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                        pSup->rowEntryInfoOffset);
5
54liuyao 已提交
3353 3354 3355 3356 3357 3358
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
  doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
  if (pInfo->pDelRes->info.rows > 0) {
    return pInfo->pDelRes;
  }
X
Xiaoyu Wang 已提交
3359
  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
5
54liuyao 已提交
3360 3361 3362
  return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
}

3363 3364 3365 3366
int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo));
}

X
Xiaoyu Wang 已提交
3367 3368 3369 3370 3371 3372
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                SExecTaskInfo* pTaskInfo) {
  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
  SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
  SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
3373
  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
5
54liuyao 已提交
3374

X
Xiaoyu Wang 已提交
3375 3376
  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
5
54liuyao 已提交
3377 3378 3379 3380
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }

3381 3382
  SExprSupp* pSup = &pOperator->exprSupp;

X
Xiaoyu Wang 已提交
3383
  int32_t    numOfCols = 0;
5
54liuyao 已提交
3384 3385 3386 3387
  SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
  initResultSizeInfo(pOperator, 4096);
3388
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
X
Xiaoyu Wang 已提交
3389 3390
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
5
54liuyao 已提交
3391 3392
      .calTrigger = pStateNode->window.triggerType,
      .maxTs = INT64_MIN,
X
Xiaoyu Wang 已提交
3393
  };
5
54liuyao 已提交
3394
  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
3395

3396
  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
5
54liuyao 已提交
3397 3398 3399 3400
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

3401
  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pSup->pCtx, numOfCols);
5
54liuyao 已提交
3402 3403 3404 3405 3406 3407 3408 3409 3410
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
  if (pInfo->pDummyCtx == NULL) {
    goto _error;
  }

3411
  initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols);
5
54liuyao 已提交
3412 3413 3414 3415 3416 3417 3418 3419 3420 3421
  pInfo->primaryTsIndex = tsSlotId;
  pInfo->order = TSDB_ORDER_ASC;
  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
  pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
  pInfo->pDelRes = createOneDataBlock(pResBlock, false);
  blockDataEnsureCapacity(pInfo->pDelRes, 64);
  pInfo->pChildren = NULL;

  pOperator->name = "StreamStateAggOperator";
3422
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
5
54liuyao 已提交
3423 3424
  pOperator->blocking = true;
  pOperator->status = OP_NOT_OPENED;
3425 3426
  pOperator->exprSupp.numOfExprs = numOfCols;
  pOperator->exprSupp.pExprInfo = pExprInfo;
5
54liuyao 已提交
3427 3428
  pOperator->pTaskInfo = pTaskInfo;
  pOperator->info = pInfo;
X
Xiaoyu Wang 已提交
3429 3430 3431
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL,
                                         destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
  initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType);
5
54liuyao 已提交
3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444
  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
  return pOperator;

_error:
  destroyStreamStateOperatorInfo(pInfo, numOfCols);
  taosMemoryFreeClear(pInfo);
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}
3445 3446 3447 3448

typedef struct SMergeIntervalAggOperatorInfo {
  SIntervalAggOperatorInfo intervalAggOperatorInfo;

S
shenglian zhou 已提交
3449 3450 3451
  bool         hasGroupId;
  uint64_t     groupId;
  SSDataBlock* prefetchedBlock;
3452
  bool         inputBlocksFinished;
3453 3454 3455
} SMergeIntervalAggOperatorInfo;

void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) {
3456 3457 3458 3459
  SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param;
  destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput);
}

3460
static int32_t outputMergeIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock, TSKEY wstartTs) {
S
shenglian zhou 已提交
3461 3462 3463
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
  SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
3464 3465

  SExprSupp* pSup = &pOperatorInfo->exprSupp;
S
shenglian zhou 已提交
3466
  bool                           ascScan = (iaInfo->order == TSDB_ORDER_ASC);
3467

3468 3469 3470 3471
  SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId);
  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf,
                                                            GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
  ASSERT(p1 != NULL);
3472

3473 3474
  finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pSup->pCtx, pSup->pExprInfo,
                                       pSup->numOfExprs, pSup->rowEntryInfoOffset, pResultBlock,
3475 3476
                                       pTaskInfo);
  taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
3477

3478
  return 0;
3479 3480 3481
}

static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
S
shenglian zhou 已提交
3482 3483 3484
                                   int32_t scanFlag, SSDataBlock* pResultBlock) {
  SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
3485 3486

  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
3487
  SExprSupp* pSup = &pOperatorInfo->exprSupp;
3488 3489

  int32_t     startPos = 0;
3490
  int32_t     numOfOutput = pSup->numOfExprs;
3491
  int64_t*    tsCols = extractTsCol(pBlock, iaInfo);
3492 3493 3494 3495
  uint64_t    tableGroupId = pBlock->info.groupId;
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

3496 3497 3498
  STimeWindow win;
  win.skey = blockStartTs;
  win.ekey = taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1;
3499

3500
  //TODO: remove the hash table usage (groupid + winkey => result row position)
3501
  int32_t ret =
3502 3503
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                             numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
3504 3505 3506 3507
  if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
    longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
  }

3508 3509 3510
  TSKEY currTs = blockStartTs;
  TSKEY currPos = startPos;
  STimeWindow currWin = win;
3511
  while (1) {
3512 3513
    ++currPos;
    if (currPos >= pBlock->info.rows) {
3514 3515
      break;
    }
3516 3517 3518 3519
    if (tsCols[currPos] == currTs) {
      continue;
    } else {
      updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
3520
      doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos,
3521 3522 3523 3524 3525 3526 3527 3528
                       currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->order);

      outputMergeIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs);

      currTs = tsCols[currPos];
      currWin.skey = currTs;
      currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1;
      startPos = currPos;
3529 3530
      ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
3531 3532 3533
      if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
      }
3534 3535
    }
  }
3536
  updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
3537
  doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos,
3538
                   currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->order);
3539

3540
  outputMergeIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs);
3541 3542 3543
}

static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
S
shenglian zhou 已提交
3544
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
3545 3546

  SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info;
S
shenglian zhou 已提交
3547
  SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
3548 3549 3550 3551
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
  }

3552
  SExprSupp* pSup = &pOperator->exprSupp;
3553
  SSDataBlock* pRes = iaInfo->binfo.pRes;
3554
  blockDataCleanup(pRes);
3555
  blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);
3556

3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567
  if (!miaInfo->inputBlocksFinished) {
    SOperatorInfo* downstream = pOperator->pDownstream[0];
    int32_t        scanFlag = MAIN_SCAN;
    while (1) {
      SSDataBlock* pBlock = NULL;
      if (miaInfo->prefetchedBlock == NULL) {
        pBlock = downstream->fpSet.getNextFn(downstream);
      } else {
        pBlock = miaInfo->prefetchedBlock;
        miaInfo->groupId = pBlock->info.groupId;
      }
3568

3569 3570 3571 3572
      if (pBlock == NULL) {
        miaInfo->inputBlocksFinished = true;
        break;
      }
3573

3574 3575 3576 3577 3578 3579 3580
      if (!miaInfo->hasGroupId) {
        miaInfo->hasGroupId = true;
        miaInfo->groupId = pBlock->info.groupId;
      } else if (miaInfo->groupId != pBlock->info.groupId) {
        miaInfo->prefetchedBlock = pBlock;
        break;
      }
3581

3582
      getTableScanInfo(pOperator, &iaInfo->order, &scanFlag);
3583
      setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->order, scanFlag, true);
3584
      doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);
3585

3586 3587 3588 3589 3590 3591
      if (pRes->info.rows >= pOperator->resultInfo.threshold) {
        break;
      }
    }

    pRes->info.groupId = miaInfo->groupId;
3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603
  }

  if (pRes->info.rows == 0) {
    doSetOperatorCompleted(pOperator);
  }

  size_t rows = pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
}

SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
S
shenglian zhou 已提交
3604 3605
                                               SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
                                               SExecTaskInfo* pTaskInfo) {
3606
  SMergeIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo));
S
shenglian zhou 已提交
3607
  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
3608 3609 3610 3611
  if (miaInfo == NULL || pOperator == NULL) {
    goto _error;
  }

S
shenglian zhou 已提交
3612
  SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
3613
  SExprSupp* pSup = &pOperator->exprSupp;
3614

3615 3616 3617
  iaInfo->win       = pTaskInfo->window;
  iaInfo->order     = TSDB_ORDER_ASC;
  iaInfo->interval  = *pInterval;
3618 3619
  iaInfo->execModel = pTaskInfo->execModel;
  iaInfo->primaryTsIndex = primaryTsSlotId;
3620 3621 3622 3623

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);

3624 3625
  int32_t code = initAggInfo(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&iaInfo->binfo, pResBlock);
3626

3627
  initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win);
3628

3629
  iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, iaInfo);
3630 3631
  if (iaInfo->timeWindowInterpo) {
    iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition));
3632 3633
  }

3634
  if (code != TSDB_CODE_SUCCESS) {
3635 3636 3637
    goto _error;
  }

3638
  initResultRowInfo(&iaInfo->binfo.resultRowInfo);
3639 3640 3641 3642 3643

  pOperator->name = "TimeMergeIntervalAggOperator";
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL;
  pOperator->blocking = false;
  pOperator->status = OP_NOT_OPENED;
3644
  pOperator->exprSupp.pExprInfo = pExprInfo;
3645
  pOperator->pTaskInfo = pTaskInfo;
3646
  pOperator->exprSupp.numOfExprs = numOfCols;
3647
  pOperator->info = miaInfo;
3648

3649
  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL,
3650
                                         destroyMergeIntervalOperatorInfo, NULL, NULL, NULL);
3651 3652 3653 3654 3655 3656 3657 3658 3659

  code = appendDownstream(pOperator, &downstream, 1);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }

  return pOperator;

_error:
3660 3661
  destroyMergeIntervalOperatorInfo(miaInfo, numOfCols);
  taosMemoryFreeClear(miaInfo);
3662 3663 3664 3665
  taosMemoryFreeClear(pOperator);
  pTaskInfo->code = code;
  return NULL;
}