[TD-225]

645514c4 · Haojun Liao · b8ffb3b0 · 645514c4 · 645514c4 · 645514c4
5 changed file
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -820,7 +820,7 @@ int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQu
  const char* msg1 = "sliding value no larger than the interval value";
  const char* msg2 = "sliding value can not less than 1% of interval value";
  const char* msg3 = "does not support sliding when interval is natural month/year";
-  const char* msg4 = "sliding not support yet in ordinary query";
+//  const char* msg4 = "sliding not support yet in ordinary query";

  const static int32_t INTERVAL_SLIDING_FACTOR = 100;
  SSqlCmd* pCmd = &pSql->cmd;
@@ -856,9 +856,9 @@ int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQu
    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
  }

-  if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) {
-    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
-  }
+//  if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) {
+//    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
+//  }

  return TSDB_CODE_SUCCESS;
 }

--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -170,8 +170,6 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) {
  tw->ekey -= 1;
 }

-#define GET_NEXT_TIMEWINDOW(_q, tw) getNextTimeWindow((_q), (tw))
-
 #define SET_STABLE_QUERY_OVER(_q) ((_q)->tableIndex = (int32_t)((_q)->tableqinfoGroupInfo.numOfTables))
 #define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables))

@@ -827,7 +825,7 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow
    TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) {
  SQuery *pQuery = pRuntimeEnv->pQuery;

-  GET_NEXT_TIMEWINDOW(pQuery, pNext);
+  getNextTimeWindow(pQuery, pNext);

  // next time window is not in current block
  if ((pNext->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
@@ -1342,9 +1340,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
      int32_t     index = pWindowResInfo->curIndex;

      while (1) {
-        GET_NEXT_TIMEWINDOW(pQuery, &nextWin);
+        getNextTimeWindow(pQuery, &nextWin);
        if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
-            (nextWin.skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
+            (nextWin.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
          break;
        }

@@ -2202,7 +2200,7 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
    }

    while(1) {
-      GET_NEXT_TIMEWINDOW(pQuery, &w);
+      getNextTimeWindow(pQuery, &w);
      if (w.skey > pBlockInfo->window.ekey) {
        break;
      }
@@ -2221,7 +2219,7 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
    }

    while(1) {
-      GET_NEXT_TIMEWINDOW(pQuery, &w);
+      getNextTimeWindow(pQuery, &w);
      if (w.ekey < pBlockInfo->window.skey) {
        break;
      }
@@ -2536,7 +2534,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
    setQueryStatus(pQuery, QUERY_COMPLETED);
  }

-  if (QUERY_IS_INTERVAL_QUERY(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) {
+  if (QUERY_IS_INTERVAL_QUERY(pQuery) && (IS_MASTER_SCAN(pRuntimeEnv)|| pRuntimeEnv->scanFlag == REPEAT_SCAN)) {
    if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
      closeAllTimeWindow(&pRuntimeEnv->windowResInfo);
      pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1;  // point to the last time window
@@ -4354,14 +4352,17 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
    STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery);

    while (pQuery->limit.offset > 0) {
+      STimeWindow tw = win;
+
      if ((win.ekey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
          (win.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
        pQuery->limit.offset -= 1;
        pWindowResInfo->prevSKey = win.skey;
-      }

-      STimeWindow tw = win;
-      GET_NEXT_TIMEWINDOW(pQuery, &tw);
+        getNextTimeWindow(pQuery, &tw);
+      } else { // current window does not ended in current data block, try next data block
+        getNextTimeWindow(pQuery, &tw);
+      }

      if (pQuery->limit.offset == 0) {
        if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
@@ -4414,6 +4415,49 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
        SArray *         pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL);
        SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);

+        if ((win.ekey > blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
+            (win.ekey < blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
+          pQuery->limit.offset -= 1;
+        }
+
+        if (pQuery->limit.offset == 0) {
+          if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
+              (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
+            // load the data block and check data remaining in current data block
+            // TODO optimize performance
+            SArray *         pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL);
+            SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
+
+            tw = win;
+            int32_t startPos =
+                getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
+            assert(startPos >= 0);
+
+            // set the abort info
+            pQuery->pos = startPos;
+
+            // reset the query start timestamp
+            pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos];
+            pQuery->window.skey = pTableQueryInfo->win.skey;
+            *start = pTableQueryInfo->win.skey;
+
+            pWindowResInfo->prevSKey = tw.skey;
+            int32_t index = pRuntimeEnv->windowResInfo.curIndex;
+
+            int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock);
+            pRuntimeEnv->windowResInfo.curIndex = index;  // restore the window index
+
+            qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64,
+                   GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey);
+
+            return true;
+          } else { // do nothing
+            *start = tw.skey;
+            pQuery->window.skey = tw.skey;
+            pWindowResInfo->prevSKey = tw.skey;
+            return true;
+          }
+        } else {
          tw = win;
          int32_t startPos =
              getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
@@ -4424,6 +4468,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
          pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos];
          pWindowResInfo->prevSKey = tw.skey;
          win = tw;
+        }
      } else {
        break;  // offset is not 0, and next time window begins or ends in the next block.
      }
@@ -6926,7 +6971,6 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co

  if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
    // here current thread hold the refcount, so it is safe to free tsdbQueryHandle.
-//    doFreeQueryHandle(pQInfo);
    *continueExec = false;
    (*pRsp)->completed = 1;  // notify no more result to client
  } else {

--- a/src/query/src/sql.c
+++ b/src/query/src/sql.c
@@ -2628,13 +2628,13 @@ static void yy_reduce(
 {yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;}
        break;
      case 177: /* limit_opt ::= LIMIT signed */
-{printf("aa1, %d\n", yymsp[0].minor.yy207); yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-1].minor.yy216.offset = 0;}
+{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-1].minor.yy216.offset = 0;}
        break;
      case 178: /* limit_opt ::= LIMIT signed OFFSET signed */
-{printf("aa2\n, %d\n", yymsp[-2].minor.yy207); yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
+{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
        break;
      case 179: /* limit_opt ::= LIMIT signed COMMA signed */
-{printf("aa3\n, %d\n", yymsp[-2].minor.yy207); yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
+{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
        break;
      case 181: /* slimit_opt ::= SLIMIT signed */
 {yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-1].minor.yy216.offset = 0;}

--- a/src/rpc/src/rpcTcp.c
+++ b/src/rpc/src/rpcTcp.c
@@ -373,8 +373,8 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
    tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", 
            pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds);
  } else {
-    taosCloseSocket(fd);
    tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
+    taosCloseSocket(fd);
  }

  return pFdObj;

--- a/tests/script/general/parser/sliding.sim
+++ b/tests/script/general/parser/sliding.sim
@@ -4,6 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/cfg.sh -n dnode1 -c debugFlag -v 135
 system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135
+system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4
 system sh/exec.sh -n dnode1 -s start
 sleep 1000
 sql connect
@@ -28,7 +29,7 @@ $mt = $mtPrefix . $i

 sql drop database if exists $db -x step1
 step1:
-sql create database if not exists $db maxtables 4 keep 36500
+sql create database if not exists $db  keep 36500
 sql use $db
 sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))

@@ -169,19 +170,20 @@ if $data94 != 90 then
 endi

 sql select count(c2),last(c4) from sliding_tb0 interval(30s) sliding(10s) order by ts asc;
-if $row != 30 then
+if $row != 32 then
  return -1
 endi

-if $data00 != @00-01-01 00:00:00.000@ then
+if $data00 != @99-12-31 23:59:40.000@ then
+  print expect 12-31 23:59:40.000, actual: $data00
  return -1
 endi

-if $data01 != 1000 then
+if $data01 != 334 then
  return -1
 endi

-if $data02 != 99 then
+if $data02 != 33 then
  return -1
 endi

@@ -304,11 +306,11 @@ if $data13 != 9.810708435 then
 endi

 sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 1;
-if $row != 14 then
+if $row != 15 then
 return -1
 endi

-if $data00 != @00-01-01 00:00:20.000@ then
+if $data00 != @00-01-01 00:00:00.000@ then
 return -1
 endi

@@ -316,7 +318,7 @@ if $data01 != 1000 then
 return -1
 endi

-if $data02 != 66 then
+if $data02 != 99 then
 return -1
 endi

@@ -324,7 +326,7 @@ if $data03 != 28.866070048 then
 return -1
 endi

-if $data90 != @00-01-01 00:03:20.000@ then
+if $data90 != @00-01-01 00:03:00.000@ then
 return -1
 endi

@@ -332,113 +334,158 @@ if $data91 != 1000 then
 return -1
 endi

-if $data92 != 66 then
+if $data92 != 99 then
 return -1
 endi

-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 14;
-if $row != 1 then
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 5;
+if $row != 11 then
 return -1
 endi

-sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by ts desc;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 6;
 if $row != 10 then
 return -1
 endi

-#00-01-01 00:04:30.000|                   10|       0|              0.000000000|              0.000000000|
-if $data00 != @00-01-01 00:04:30.000@ then
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 7;
+if $row != 9 then
 return -1
 endi

-if $data01 != 10 then
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 8;
+if $row != 8 then
 return -1
 endi

-if $data02 != 0 then
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 9;
+if $row != 7 then
 return -1
 endi

-if $data03 != 0.000000000 then
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 10;
+if $row != 6 then
 return -1
 endi

-sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by ts desc limit 1 offset 15;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 11;
+if $row != 5 then
+ return -1
+endi
+
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 12;
+if $row != 4 then
+ return -1
+endi
+
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 13;
+if $row != 3 then
+ return -1
+endi
+
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 14;
+if $row != 2 then
+ return -1
+endi
+
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 15;
+if $row != 1 then
+ return -1
+endi
+
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 16;
 if $row != 0 then
 return -1
 endi

-sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10a) order by ts asc limit 1000;
-if $row != 100 then
+sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by ts desc;
+if $row != 10 then
 return -1
 endi

-if $data00 != @00-01-01 00:00:00.000@ then
+#00-01-01 00:04:30.000|                   10|       0|              0.000000000|              0.000000000|
+if $data00 != @00-01-01 00:04:30.000@ then
 return -1
 endi

-if $data02 != 9.521904571 then
+if $data01 != 10 then
 return -1
 endi

-if $data05 != 33 then
+if $data02 != 0 then
 return -1
 endi

-if $data10 != @00-01-01 00:00:00.010@ then
+if $data03 != 0.000000000 then
 return -1
 endi

-if $data12 != 9.521904571 then
+sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by ts desc limit 1 offset 15;
+if $row != 1 then
 return -1
 endi

-if $data15 != 33 then
+sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by ts desc limit 1 offset 16;
+if $row != 0 then
 return -1
 endi

-if $data95 != 33 then
+sql select count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) order by ts desc limit 10 offset 2;
+if $data00 != @00-01-01 00:04:59.910@ then
  return -1
 endi

-sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10a) order by ts desc limit 1000;
-if $row != 100 then
+sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10s) order by ts asc limit 1000;
+if $row != 3 then
 return -1
 endi

-if $data00 != @00-01-01 00:00:00.990@ then
+if $data00 != @99-12-31 23:59:40.000@ then
 return -1
 endi

-if $data01 != 1 then
+if $data02 != 9.521904571 then
+ return -1
+endi
+
+if $data05 != 33 then
+ return -1
+endi
+
+if $data10 != @99-12-31 23:59:50.000@ then
+ return -1
+endi
+
+if $data12 != 9.521904571 then
 return -1
 endi

-if $data02 != 0.000000000 then
+if $data15 != 33 then
 return -1
 endi

-if $data03 != 1 then
+if $data25 != 33 then
 return -1
 endi

-if $data90 != @00-01-01 00:00:00.900@ then
+sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10s) order by ts desc limit 1000;
+if $row != 1 then
 return -1
 endi

-if $data91 != 4 then
+if $data00 != @99-12-31 23:59:40.000@ then
 return -1
 endi

-if $data92 != 1.118033989 then
+if $data01 != 33 then
 return -1
 endi

-if $data93 != 4 then
+if $data02 != 9.521904571 then
 return -1
 endi

-if $data94 != 30.00000 then
+if $data03 != 33 then
 return -1
 endi

@@ -457,5 +504,7 @@ sql_error select sum(c1) from sliding_tb0 interval(0)  sliding(0);
 sql_error select sum(c1) from sliding_tb0 interval(0m)  sliding(0m);
 sql_error select sum(c1) from sliding_tb0 interval(m)  sliding(m);
 sql_error select sum(c1) from sliding_tb0 sliding(4m);
+sql_error select count(*) from sliding_tb0 interval(1s) sliding(10s);
+sql_error select count(*) from sliding_tb0 interval(10s) sliding(10a);

 system sh/exec.sh -n dnode1 -s stop -x SIGINT