Merge pull request #16658 from taosdata/feature/TD-18820

feat(stream): refactor single interval

source/libs/executor/inc/executorimpl.h

@@ -591,6 +591,24 @@ typedef struct SMergeAlignedIntervalAggOperatorInfo {
   SNode*       pCondition;
 } SMergeAlignedIntervalAggOperatorInfo;
 
+typedef struct SStreamIntervalOperatorInfo {
+  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
+  SOptrBasicInfo     binfo;              // basic info
+  SAggSupporter      aggSup;             // aggregate supporter
+  SExprSupp          scalarSupp;         // supporter for perform scalar function
+  SGroupResInfo      groupResInfo;       // multiple results build supporter
+  SInterval          interval;           // interval info
+  int32_t            primaryTsIndex;     // primary time stamp slot id from result of downstream operator.
+  STimeWindowAggSupp twAggSup;
+  bool               invertible;
+  bool               ignoreExpiredData;
+  SArray*            pRecycledPages;
+  SArray*            pDelWins;           // SWinRes
+  int32_t            delIndex;
+  SSDataBlock*       pDelRes;
+  bool               isFinal;
+} SStreamIntervalOperatorInfo;
+
 typedef struct SStreamFinalIntervalOperatorInfo {
   // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
   SOptrBasicInfo     binfo;           // basic info
@@ -1003,6 +1021,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
                                                   SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                        SExecTaskInfo* pTaskInfo, int32_t numOfChild);
+SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream,
+                                                SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                 SExecTaskInfo* pTaskInfo);

source/libs/executor/src/executorimpl.c

@@ -3913,7 +3913,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions,
                                           pScalarExprInfo, numOfScalarExpr, pAggNode->mergeDataBlock, pTaskInfo);
     }
-  } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
+  } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) {
     SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
 
     SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
@@ -3938,6 +3938,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode,
                                        pTaskInfo, isStream);
 
+  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { 
+    pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) {
     SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode;
     pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo);

tests/script/tsim/stream/basic1.sim

@@ -588,4 +588,38 @@ if $data00 != 5 then
   goto loop3
 endi
 
+#max,min selectivity
+sql create database test3  vgroups 1;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from ts1 interval(10s) ;
+
+sql insert into ts1 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts1 values(1648791222001,2,2,3);
+sleep 50
+
+$loop_count = 0
+loop3:
+sql select * from streamtST3;
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+# row 0
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop3
+endi
+
+# row 1
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop3
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stream/distributeInterval0.sim

@@ -198,7 +198,7 @@ endi
 sql select  _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b)  c4, min(c) c5, avg(d) from st interval(10s);
 
 
-sql create database test1  vgroups 1;
+sql create database test1  vgroups 4;
 sql use test1;
 sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
 sql create table ts1 using st tags(1,1,1);
@@ -232,4 +232,43 @@ if $data11 != 2 then
   goto loop2
 endi
 
+#max,min selectivity
+sql create database test3  vgroups 4;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create table ts2 using st tags(2,2,2);
+sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ;
+
+sql insert into ts1 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts1 values(1648791222001,2,2,3);
+sleep 50
+sql insert into ts2 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts2 values(1648791222001,2,2,3);
+sleep 50
+
+$loop_count = 0
+loop3:
+sql select * from streamtST3;
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+# row 0
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop3
+endi
+
+# row 1
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop3
+endi
+
 system sh/stop_dnodes.sh

tests/script/tsim/stream/partitionbyColumn0.sim

@@ -24,7 +24,7 @@ sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop0:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -48,7 +48,7 @@ sql insert into t1 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop1:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -71,7 +71,7 @@ sql insert into t1 values(1648791213000,2,2,3,1.0);
 $loop_count = 0
 
 loop2:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -97,7 +97,7 @@ sql insert into t1 values(1648791213002,1,2,3,1.0);
 $loop_count = 0
 
 loop3:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -134,7 +134,7 @@ sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (16
 $loop_count = 0
 
 loop4:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -208,7 +208,7 @@ sql insert into t1 values(1648791213001,1,2,3,2.0);
 $loop_count = 0
 
 loop5:
-sleep 100
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -229,7 +229,7 @@ sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
 $loop_count = 0
 
 loop6:
-sleep 100
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -294,7 +294,7 @@ sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop7:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -318,7 +318,7 @@ sql insert into t2 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop8:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -342,7 +342,7 @@ sql insert into t2 values(1648791213000,2,2,3,1.0);
 $loop_count = 0
 
 loop9:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -372,7 +372,7 @@ sql insert into t2 values(1648791213002,1,2,3,1.0);
 $loop_count = 0
 
 loop10:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -414,7 +414,7 @@ sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (16
 $loop_count = 0
 
 loop11:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -492,7 +492,7 @@ sql insert into t4 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop13:
-sleep 100
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -534,7 +534,7 @@ sql insert into t1 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop14:
-sleep 100
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1

tests/script/tsim/stream/partitionbyColumn1.sim

@@ -24,11 +24,11 @@ sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop0:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -45,12 +45,14 @@ endi
 
 sql insert into t1 values(1648791213000,1,2,3,1.0);
 
+$loop_count = 0
+
 loop1:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -66,12 +68,14 @@ endi
 
 sql insert into t1 values(1648791213000,2,2,3,1.0);
 
+$loop_count = 0
+
 loop2:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -90,12 +94,14 @@ sql insert into t1 values(1648791213001,2,2,3,1.0);
 sql insert into t1 values(1648791213002,2,2,3,1.0);
 sql insert into t1 values(1648791213002,1,2,3,1.0);
 
+$loop_count = 0
+
 loop3:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -125,12 +131,14 @@ sql insert into t1 values(1648791223002,3,2,3,1.0);
 sql insert into t1 values(1648791223003,3,2,3,1.0);
 sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
 
+$loop_count = 0
+
 loop4:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -199,11 +207,11 @@ sql insert into t1 values(1648791213001,1,2,3,2.0);
 $loop_count = 0
 
 loop5:
-sleep 300
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -217,12 +225,14 @@ sql insert into t1 values(1648791223001,1,2,5,2.0);
 sql insert into t1 values(1648791223002,1,2,5,2.0);
 sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
 
+$loop_count = 0
+
 loop6:
-sleep 300
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -282,11 +292,11 @@ sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop7:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -303,12 +313,14 @@ endi
 sql insert into t1 values(1648791213000,1,2,3,1.0);
 sql insert into t2 values(1648791213000,1,2,3,1.0);
 
+$loop_count = 0
+
 loop8:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -324,12 +336,15 @@ endi
 
 sql insert into t1 values(1648791213000,2,2,3,1.0);
 sql insert into t2 values(1648791213000,2,2,3,1.0);
+
+$loop_count = 0
+
 loop9:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -352,12 +367,14 @@ sql insert into t2 values(1648791213001,2,2,3,1.0);
 sql insert into t2 values(1648791213002,2,2,3,1.0);
 sql insert into t2 values(1648791213002,1,2,3,1.0);
 
+$loop_count = 0
+
 loop10:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -373,7 +390,7 @@ endi
 
 if $data11 != 2 thenloop4
   print =====data11=$data11
-  goto loop3
+  goto loop10
 endi
 
 if $data12 != 1 then
@@ -392,12 +409,14 @@ sql insert into t2 values(1648791223002,3,2,3,1.0);
 sql insert into t2 values(1648791223003,3,2,3,1.0);
 sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
 
+$loop_count = 0
+
 loop11:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -470,17 +489,17 @@ sql insert into t4 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop13:
-sleep 300
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
 if $rows != 2 then
   print =====rows=$rows
-  goto loop14
+  goto loop13
 endi
 
 if $data01 != 1 then
@@ -495,12 +514,12 @@ endi
 
 if $data11 != 3 then
   print =====data11=$data11
-  goto loop11
+  goto loop13
 endi
 
 if $data12 != 2 then
   print =====data12=$data12
-  goto loop11
+  goto loop13
 endi
 
 sql insert into t4 values(1648791213000,2,2,3,1.0);
@@ -509,12 +528,14 @@ sql insert into t1 values(1648791233000,2,2,3,1.0);
 
 sql insert into t1 values(1648791213000,1,2,3,1.0); 
 
+$loop_count = 0
+
 loop14:
-sleep 300
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 

tests/script/tsim/stream/partitionbyColumn2.sim

@@ -40,6 +40,8 @@ endi
 
 sql insert into t1 values(1648791213000,1,1,3,1.0);
 
+$loop_count = 0
+
 loop1:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -61,6 +63,8 @@ endi
 
 sql insert into t1 values(1648791213000,2,1,3,1.0);
 
+$loop_count = 0
+
 loop2:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -85,6 +89,8 @@ sql insert into t1 values(1648791213001,2,1,3,1.0);
 sql insert into t1 values(1648791213002,2,1,3,1.0);
 sql insert into t1 values(1648791213002,1,1,3,1.0);
 
+$loop_count = 0
+
 loop3:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -120,6 +126,8 @@ sql insert into t1 values(1648791223002,3,2,3,1.0);
 sql insert into t1 values(1648791223003,3,2,3,1.0);
 sql insert into t1 values(1648791213001,1,1,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
 
+$loop_count = 0
+
 loop4:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -212,6 +220,8 @@ sql insert into t1 values(1648791223001,1,2,2,5);
 sql insert into t1 values(1648791223002,1,2,2,6);
 sql insert into t1 values(1648791213001,1,1,1,7) (1648791223002,1,1,2,8);
 
+$loop_count = 0
+
 loop6:
 sleep 300
 sql select * from streamt1 order by c1, c4, c2, c3;