Merge branch '3.0' into feature/3_liaohj

source/libs/executor/src/executorimpl.c

@@ -3323,6 +3323,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       if (pLimitInfo->remainGroupOffset > 0) {
         if (pLimitInfo->currentGroupId == 0 || pLimitInfo->currentGroupId == pBlock->info.groupId) {  // it is the first group
           pLimitInfo->currentGroupId = pBlock->info.groupId;
+          ASSERT(pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM);
           continue;
         } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) {
           // now it is the data from a new group
@@ -3331,6 +3332,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
 
           // ignore data block in current group
           if (pLimitInfo->remainGroupOffset > 0) {
+            ASSERT(pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM);
             continue;
           }
         }
@@ -3375,10 +3377,12 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       if (pLimitInfo->remainOffset >= pInfo->pRes->info.rows) {
         pLimitInfo->remainOffset -= pInfo->pRes->info.rows;
         blockDataCleanup(pInfo->pRes);
+        ASSERT(pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM);
         continue;
       } else if (pLimitInfo->remainOffset < pInfo->pRes->info.rows && pLimitInfo->remainOffset > 0) {
         blockDataTrimFirstNRows(pInfo->pRes, pLimitInfo->remainOffset);
         pLimitInfo->remainOffset = 0;
+        ASSERT(pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM);
       }
 
       // check for the limitation in each group
@@ -3386,6 +3390,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
           pLimitInfo->numOfOutputRows + pInfo->pRes->info.rows >= pLimitInfo->limit.limit) {
         int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows);
         blockDataKeepFirstNRows(pInfo->pRes, keepRows);
+        ASSERT(pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM);
         if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) {
           pOperator->status = OP_EXEC_DONE;
         }
@@ -3395,7 +3400,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       break;
     }
 
-    if (pProjectInfo->mergeDataBlocks) {
+    if (pProjectInfo->mergeDataBlocks && pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM) {
       if (pRes->info.rows > 0) {
         pFinalRes->info.groupId = pRes->info.groupId;
         pFinalRes->info.version = pRes->info.version;
@@ -3420,8 +3425,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
           continue;
         }
       }
-
-      // no results generated
+      
       break;
     }
   }

tests/pytest/crash_gen/crash_gen_main.py

@@ -1327,6 +1327,8 @@ class Task():
 
                 # TDengine 3.0 Error Codes:
                 0x0333, # Object is creating # TODO: this really is NOT an acceptable error
+                0x0369, # Tag already exists
+                0x0388, # Database not exist
                 0x03A0, # STable already exists
                 0x03A1, # STable [does] not exist
                 0x03AA, # Tag already exists

tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py

+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def __init__(self):
+        self.vgroups    = 4
+        self.ctbNum     = 100
+        self.rowsPerTbl = 1000
+        
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     100,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   100,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        tdLog.info("flush db to let data falls into the disk")
+        tdSql.query("flush database %s"%(paraDict['dbName']))
+        return
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     100,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   500,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  5,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        # tdSql.query(queryString)        
+        # expectRowsList.append(tdSql.getRows())
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        # after start consume, continue insert some data
+        paraDict['batchNum']   = 100
+        paraDict['startTs'] = paraDict['startTs'] + self.rowsPerTbl
+        pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
+        
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+
+        pInsertThread.join()
+        
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+        
+        tdLog.info("wait the consume result")         
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+                
+        tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
+        if expectRowsList[0] != resultList[0]:
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        # tmqCom.checkFileContent(consumerId, queryString) 
+
+        tdSql.query("flush database %s"%(paraDict['dbName']))
+        
+        for i in range(len(topicNameList)):
+            tmqCom.waitSubscriptionExit(tdSql,topicNameList[i])    
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def tmqCase2(self):
+        tdLog.printNoPrefix("======== test case 2: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     100,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   500,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+        totalRowsInserted = expectRowsList[0]
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 1
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3)
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor 0")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result") 
+        
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        actConsumeRows = resultList[0]
+                
+        tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(actConsumeRows, expectrowcnt, totalRowsInserted))
+        if not (expectrowcnt <= actConsumeRows and totalRowsInserted >= actConsumeRows):
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+        
+        # reinit consume info, and start tmq_sim, then check consume result
+        tmqCom.initConsumerTable()
+        consumerId   = 2
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3)
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        tdLog.info("start consume processor 1")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result") 
+        
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        actConsumeRows = resultList[0]
+        tdLog.info("act consume rows: %d, expect rows: %d, act insert rows: %d"%(actConsumeRows, expectrowcnt, totalRowsInserted))    
+        if not ((actConsumeRows >= expectrowcnt) and (totalRowsInserted > actConsumeRows)):
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+            
+        for i in range(len(topicNameList)):
+            tmqCom.waitSubscriptionExit(tdSql,topicNameList[i])    
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 2 end ...... ")
+
+    def run(self):
+        tdSql.prepare()
+        self.prepareTestEnv()
+        self.tmqCase1()
+        self.tmqCase2()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/dataFromTsdbNWal.py

@@ -17,8 +17,8 @@ from tmqCommon import *
 
 class TDTestCase:
     def __init__(self):
-        self.vgroups    = 1
-        self.ctbNum     = 100
+        self.vgroups    = 4
+        self.ctbNum     = 1
         self.rowsPerTbl = 10000
         
     def init(self, conn, logSql):
@@ -38,9 +38,9 @@ class TDTestCase:
                     'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
                     'ctbPrefix':  'ctb',
                     'ctbStartIdx': 0,
-                    'ctbNum':     100,
+                    'ctbNum':     1,
                     'rowsPerTbl': 10000,
-                    'batchNum':   3000,
+                    'batchNum':   100,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
                     'pollDelay':  10,
                     'showMsg':    1,
@@ -85,7 +85,7 @@ class TDTestCase:
                     'rowsPerTbl': 10000,
                     'batchNum':   100,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    'pollDelay':  3,
+                    'pollDelay':  5,
                     'showMsg':    1,
                     'showRow':    1,
                     'snapshot':   1}
@@ -117,17 +117,16 @@ class TDTestCase:
         keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
         tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
 
-        tdLog.info("start consume processor")
-        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
-
         # after start consume, continue insert some data
         paraDict['batchNum']   = 100
         paraDict['startTs'] = paraDict['startTs'] + self.rowsPerTbl
-        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
-                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
-                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
+        
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
 
-        # 
+        pInsertThread.join()
+        
         tdSql.query(queryString)        
         expectRowsList.append(tdSql.getRows())
         
@@ -135,15 +134,16 @@ class TDTestCase:
         expectRows = 1
         resultList = tmqCom.selectConsumeResult(expectRows)
                 
-                
         tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
         if expectRowsList[0] != resultList[0]:
             tdLog.exit("%d tmq consume rows error!"%consumerId)
 
         tmqCom.checkFileContent(consumerId, queryString) 
 
-        time.sleep(10)        
+        tdSql.query("flush database %s"%(paraDict['dbName']))
+
         for i in range(len(topicNameList)):
+            tmqCom.waitSubscriptionExit(tdSql,topicNameList[i])   
             tdSql.query("drop topic %s"%topicNameList[i])
 
         tdLog.printNoPrefix("======== test case 1 end ...... ")
@@ -204,13 +204,12 @@ class TDTestCase:
         
         expectRows = 1
         resultList = tmqCom.selectConsumeResult(expectRows)
-        
-        if not (expectrowcnt <= resultList[0] and totalRowsInserted >= resultList[0]):
-            tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(resultList[0], expectrowcnt, totalRowsInserted))
+        actConsumeRows = resultList[0]
+                
+        tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(actConsumeRows, expectrowcnt, totalRowsInserted))
+        if not (expectrowcnt <= actConsumeRows and totalRowsInserted >= actConsumeRows):
             tdLog.exit("%d tmq consume rows error!"%consumerId)
-            
-        firstConsumeRows = resultList[0]
-
+        
         # reinit consume info, and start tmq_sim, then check consume result
         tmqCom.initConsumerTable()
         consumerId   = 2
@@ -224,15 +223,13 @@ class TDTestCase:
         expectRows = 1
         resultList = tmqCom.selectConsumeResult(expectRows)
         
-        actConsumeTotalRows = firstConsumeRows + resultList[0]
-        
-        if not (expectrowcnt >= resultList[0] and totalRowsInserted == actConsumeTotalRows):
-            tdLog.info("act consume rows, first: %d, second: %d "%(firstConsumeRows, resultList[0]))            
-            tdLog.info("and sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
+        actConsumeRows = resultList[0]
+        tdLog.info("act consume rows: %d, expect rows: %d, act insert rows: %d"%(actConsumeRows, expectrowcnt, totalRowsInserted))    
+        if not ((actConsumeRows >= expectrowcnt) and (totalRowsInserted > actConsumeRows)):
             tdLog.exit("%d tmq consume rows error!"%consumerId)
 
-        time.sleep(10)        
         for i in range(len(topicNameList)):
+            tmqCom.waitSubscriptionExit(tdSql,topicNameList[i]) 
             tdSql.query("drop topic %s"%topicNameList[i])
 
         tdLog.printNoPrefix("======== test case 2 end ...... ")
@@ -241,7 +238,7 @@ class TDTestCase:
         tdSql.prepare()
         self.prepareTestEnv()
         self.tmqCase1()
-        # self.tmqCase2()
+        self.tmqCase2()
 
     def stop(self):
         tdSql.close()

tests/system-test/7-tmq/tmqDnodeRestart.py

@@ -151,41 +151,6 @@ class TDTestCase:
         if not (totalConsumeRows == totalRowsFromQury):
             tdLog.exit("tmq consume rows error!")
 
-
-
-
-        # tdLog.info("****************************************************************************")
-        # tmqCom.initConsumerTable()
-        # consumerId     = 1
-        # expectrowcnt   = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
-        # topicList      = topicFromStb1
-        # ifcheckdata    = 0
-        # ifManualCommit = 0
-        # keyList        = 'group.id:cgrp2,\
-        #                 enable.auto.commit:true,\
-        #                 auto.commit.interval.ms:3000,\
-        #                 auto.offset.reset:earliest'
-        # tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-
-        # tdLog.info("start consume processor")
-        # tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
-        
-        # expectRows = 1
-        # resultList = tmqCom.selectConsumeResult(expectRows)
-        # totalConsumeRows = 0
-        # for i in range(expectRows):
-        #     totalConsumeRows += resultList[i]
-
-        # tdSql.query(queryString)
-        # totalRowsFromQury = tdSql.getRows()
-        
-        # tdLog.info("act consume rows: %d, act query rows: %d"%(totalConsumeRows, totalRowsFromQury))
-        # if not (totalConsumeRows == totalRowsFromQury):
-        #     tdLog.exit("tmq consume rows error!")
-
-        
-        # tdLog.info("****************************************************************************")
-
         tmqCom.waitSubscriptionExit(tdSql, topicFromStb1)
         tdSql.query("drop topic %s"%topicFromStb1)
 
@@ -259,7 +224,7 @@ class TDTestCase:
         tdLog.info("create some new child table and insert data ")
         paraDict["batchNum"] = 100
         paraDict["ctbPrefix"] = 'newCtb'
-        # tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"])
+        tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"])
         
         tdLog.info("insert process end, and start to check consume result")
         expectRows = 1

tests/system-test/7-tmq/tmqDropNtb-snapshot0.py

+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+from enum import Enum
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def __init__(self):
+        self.snapshot   = 0
+        self.vgroups    = 4
+        self.ctbNum     = 1000
+        self.rowsPerTbl = 10
+        
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+        
+    # drop some ntbs
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ntb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     1000,
+                    'rowsPerTbl': 100,
+                    'batchNum':   100,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'endTs': 0,
+                    'pollDelay':  5,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+        paraDict['snapshot'] = self.snapshot
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl        
+        
+        tmqCom.initConsumerTable()
+        tdLog.info("start create database....")
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdLog.info("start create normal tables....")
+        tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"])
+        tdLog.info("start insert data into normal tables....")
+        tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"])
+                
+        tdLog.info("create topics from database")
+        topicFromDb = 'topic_dbt'                
+        tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName']))
+        
+        if self.snapshot == 0:
+            consumerId     = 0
+        elif self.snapshot == 1:
+            consumerId     = 1
+
+        expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
+        topicList      = topicFromDb
+        ifcheckdata    = 1
+        ifManualCommit = 1
+        keyList        = 'group.id:cgrp1,\
+                        enable.auto.commit:true,\
+                        auto.commit.interval.ms:1000,\
+                        auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+
+        tmqCom.getStartConsumeNotifyFromTmqsim()  
+        tdLog.info("drop some ntables")
+        # drop 1/4 ctbls from half offset
+        paraDict["ctbStartIdx"] = paraDict["ctbStartIdx"] + int(paraDict["ctbNum"] * 1 / 2)
+        paraDict["ctbNum"] = int(paraDict["ctbNum"] / 4)
+        tmqCom.drop_ctable(tdSql, dbname=paraDict['dbName'], count=paraDict["ctbNum"], default_ctbname_prefix=paraDict["ctbPrefix"], ctbStartIdx=paraDict["ctbStartIdx"])
+        
+        tdLog.info("start to check consume result")
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+
+        tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+        
+        if not ((totalConsumeRows >= expectrowcnt * 3/4) and (totalConsumeRows < expectrowcnt)):
+            tdLog.exit("tmq consume rows error with snapshot = 0!")
+        
+        tdLog.info("wait subscriptions exit ....")      
+        tmqCom.waitSubscriptionExit(tdSql, topicFromDb)
+            
+        tdSql.query("drop topic %s"%topicFromDb)
+        tdLog.info("success dorp topic: %s"%topicFromDb)
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+    
+
+    
+    # drop some ntbs and create some new ntbs
+    def tmqCase2(self):
+        tdLog.printNoPrefix("======== test case 2: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ntb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     1000,
+                    'rowsPerTbl': 100,
+                    'batchNum':   100,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'endTs': 0,
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+        paraDict['snapshot'] = self.snapshot
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl        
+        
+        tmqCom.initConsumerTable()
+        tdLog.info("start create database....")
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdLog.info("start create normal tables....")
+        tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"])
+        tdLog.info("start insert data into normal tables....")
+        tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"])
+                
+        tdLog.info("create topics from database")
+        topicFromDb = 'topic_dbt'                
+        tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName']))
+        
+        if self.snapshot == 0:
+            consumerId     = 2
+        elif self.snapshot == 1:
+            consumerId     = 3
+
+        expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2)
+        topicList      = topicFromDb
+        ifcheckdata    = 1
+        ifManualCommit = 1
+        keyList        = 'group.id:cgrp1,\
+                        enable.auto.commit:true,\
+                        auto.commit.interval.ms:1000,\
+                        auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+
+        tmqCom.getStartConsumeNotifyFromTmqsim()  
+        tdLog.info("drop some ntables")
+        # drop 1/4 ctbls from half offset
+        paraDict["ctbStartIdx"] = paraDict["ctbStartIdx"] + int(paraDict["ctbNum"] * 1 / 2)
+        paraDict["ctbNum"] = int(paraDict["ctbNum"] / 4)
+        tmqCom.drop_ctable(tdSql, dbname=paraDict['dbName'], count=paraDict["ctbNum"], default_ctbname_prefix=paraDict["ctbPrefix"], ctbStartIdx=paraDict["ctbStartIdx"])
+                
+        tdLog.info("start create some new normal tables....")
+        paraDict["ctbPrefix"] = 'newCtb'
+        paraDict["ctbNum"] = self.ctbNum
+        tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"])
+        tdLog.info("start insert data into these new normal tables....")
+        tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"])
+        
+        tdLog.info("start to check consume result")
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+
+        tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+        
+        if not ((totalConsumeRows >= expectrowcnt / 2 * (1 + 3/4)) and (totalConsumeRows < expectrowcnt)):
+            tdLog.exit("tmq consume rows error with snapshot = 0!")
+        
+        tdLog.info("wait subscriptions exit ....")      
+        tmqCom.waitSubscriptionExit(tdSql, topicFromDb)
+            
+        tdSql.query("drop topic %s"%topicFromDb)
+        tdLog.info("success dorp topic: %s"%topicFromDb)
+        tdLog.printNoPrefix("======== test case 2 end ...... ")
+        
+    def run(self):        
+        tdLog.printNoPrefix("=============================================")
+        tdLog.printNoPrefix("======== snapshot is 0: only consume from wal")
+        self.snapshot = 0
+        self.tmqCase1()
+        self.tmqCase2()               
+        
+        # tdLog.printNoPrefix("====================================================================")
+        # tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal")
+        # self.snapshot = 1
+        # self.tmqCase1()
+        # self.tmqCase2()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/tmqDropNtb.py → tests/system-test/7-tmq/tmqDropNtb-snapshot1.py

@@ -18,7 +18,7 @@ class TDTestCase:
     def __init__(self):
         self.snapshot   = 0
         self.vgroups    = 4
-        self.ctbNum     = 100
+        self.ctbNum     = 1000
         self.rowsPerTbl = 10
         
     def init(self, conn, logSql):
@@ -39,9 +39,9 @@ class TDTestCase:
                     'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
                     'ctbPrefix':  'ntb',
                     'ctbStartIdx': 0,
-                    'ctbNum':     100,
-                    'rowsPerTbl': 1000,
-                    'batchNum':   1000,
+                    'ctbNum':     1000,
+                    'rowsPerTbl': 100,
+                    'batchNum':   100,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
                     'endTs': 0,
                     'pollDelay':  5,
@@ -125,9 +125,9 @@ class TDTestCase:
                     'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
                     'ctbPrefix':  'ntb',
                     'ctbStartIdx': 0,
-                    'ctbNum':     100,
-                    'rowsPerTbl': 1000,
-                    'batchNum':   1000,
+                    'ctbNum':     1000,
+                    'rowsPerTbl': 100,
+                    'batchNum':   100,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
                     'endTs': 0,
                     'pollDelay':  10,
@@ -203,16 +203,16 @@ class TDTestCase:
         tdLog.printNoPrefix("======== test case 2 end ...... ")
         
     def run(self):        
-        tdLog.printNoPrefix("=============================================")
-        tdLog.printNoPrefix("======== snapshot is 0: only consume from wal")
-        self.snapshot = 0
+        # tdLog.printNoPrefix("=============================================")
+        # tdLog.printNoPrefix("======== snapshot is 0: only consume from wal")
+        # self.snapshot = 0
         # self.tmqCase1()
-        self.tmqCase2()               
+        # self.tmqCase2()               
         
         tdLog.printNoPrefix("====================================================================")
         tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal")
         self.snapshot = 1
-        # self.tmqCase1()
+        self.tmqCase1()
         self.tmqCase2()
 
     def stop(self):

tests/system-test/fulltest.sh

@@ -210,7 +210,7 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py
 python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
 python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
 python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
-#python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
+python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
 python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py
 python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py
 python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py
@@ -219,12 +219,14 @@ python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py
 python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py
 python3 ./test.py -f 7-tmq/tmqDropStb.py
 python3 ./test.py -f 7-tmq/tmqDropStbCtb.py
-python3 ./test.py -f 7-tmq/tmqDropNtb.py
+python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py
+python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py
 python3 ./test.py -f 7-tmq/tmqUdf.py
 python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py
 python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py
 python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py
-
+python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py
+python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py
 # python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py
 
 #------------querPolicy  2-----------