Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
b7fae566
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1184
Star
22015
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
b7fae566
编写于
7月 30, 2022
作者:
G
Ganlin Zhao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix test cases
上级
d00dd167
变更
5
展开全部
隐藏空白更改
内联
并排
Showing
5 changed file
with
178 addition
and
178 deletion
+178
-178
tests/system-test/7-tmq/tmqDropStb.py
tests/system-test/7-tmq/tmqDropStb.py
+14
-14
tests/system-test/7-tmq/tmqError.py
tests/system-test/7-tmq/tmqError.py
+21
-21
tests/system-test/7-tmq/tmqModule.py
tests/system-test/7-tmq/tmqModule.py
+82
-82
tests/system-test/7-tmq/tmqSubscribeStb-r3.py
tests/system-test/7-tmq/tmqSubscribeStb-r3.py
+32
-32
tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py
tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py
+29
-29
未找到文件。
tests/system-test/7-tmq/tmqDropStb.py
浏览文件 @
b7fae566
...
...
@@ -47,7 +47,7 @@ class TDTestCase:
pollDelay
=
20
showMsg
=
1
showRow
=
1
showRow
=
1
hostname
=
socket
.
gethostname
()
...
...
@@ -59,7 +59,7 @@ class TDTestCase:
def
tmqCase1
(
self
):
tdLog
.
printNoPrefix
(
"======== test case 1: "
)
tdLog
.
info
(
"step 1: create database, stb, ctb and insert data"
)
tmqCom
.
initConsumerTable
(
self
.
cdbName
)
tdCom
.
create_database
(
tdSql
,
self
.
paraDict
[
"dbName"
],
self
.
paraDict
[
"dropFlag"
])
...
...
@@ -69,35 +69,35 @@ class TDTestCase:
tdCom
.
create_ctable
(
tdSql
,
dbname
=
self
.
paraDict
[
"dbName"
],
stbname
=
self
.
paraDict
[
"stbName"
],
tag_elm_list
=
self
.
paraDict
[
'tagSchema'
],
count
=
self
.
paraDict
[
"ctbNum"
],
default_ctbname_prefix
=
self
.
paraDict
[
"ctbPrefix"
])
tmqCom
.
insert_data_2
(
tdSql
,
self
.
paraDict
[
"dbName"
],
self
.
paraDict
[
"ctbPrefix"
],
self
.
paraDict
[
"ctbNum"
],
self
.
paraDict
[
"rowsPerTbl"
],
self
.
paraDict
[
"batchNum"
],
self
.
paraDict
[
"startTs"
],
self
.
paraDict
[
"ctbStartIdx"
])
# pThread1 = tmqCom.asyncInsertData(paraDict=self.paraDict)
self
.
paraDict
[
"stbName"
]
=
'stb2'
self
.
paraDict
[
"ctbPrefix"
]
=
'newctb'
self
.
paraDict
[
"batchNum"
]
=
10000
tdCom
.
create_stable
(
tdSql
,
dbname
=
self
.
paraDict
[
"dbName"
],
stbname
=
self
.
paraDict
[
"stbName"
],
column_elm_list
=
self
.
paraDict
[
"colSchema"
],
tag_elm_list
=
self
.
paraDict
[
"tagSchema"
],
count
=
1
,
default_stbname_prefix
=
self
.
paraDict
[
"stbName"
])
tdCom
.
create_ctable
(
tdSql
,
dbname
=
self
.
paraDict
[
"dbName"
],
stbname
=
self
.
paraDict
[
"stbName"
],
tag_elm_list
=
self
.
paraDict
[
'tagSchema'
],
count
=
self
.
paraDict
[
"ctbNum"
],
default_ctbname_prefix
=
self
.
paraDict
[
"ctbPrefix"
])
# tmqCom.insert_data_2(tdSql,self.paraDict["dbName"],self.paraDict["ctbPrefix"],self.paraDict["ctbNum"],self.paraDict["rowsPerTbl"],self.paraDict["batchNum"],self.paraDict["startTs"],self.paraDict["ctbStartIdx"])
pThread2
=
tmqCom
.
asyncInsertData
(
paraDict
=
self
.
paraDict
)
pThread2
=
tmqCom
.
asyncInsertData
(
paraDict
=
self
.
paraDict
)
tdLog
.
info
(
"create topics from db"
)
topicName1
=
'UpperCasetopic_%s'
%
(
self
.
paraDict
[
'dbName'
])
topicName1
=
'UpperCasetopic_%s'
%
(
self
.
paraDict
[
'dbName'
])
tdSql
.
execute
(
"create topic %s as database %s"
%
(
topicName1
,
self
.
paraDict
[
'dbName'
]))
topicList
=
topicName1
+
','
+
topicName1
keyList
=
'%s,%s,%s,%s'
%
(
self
.
groupId
,
self
.
autoCommit
,
self
.
autoCommitInterval
,
self
.
autoOffset
)
self
.
expectrowcnt
=
self
.
paraDict
[
"rowsPerTbl"
]
*
self
.
paraDict
[
"ctbNum"
]
*
2
tmqCom
.
insertConsumerInfo
(
self
.
consumerId
,
self
.
expectrowcnt
,
topicList
,
keyList
,
self
.
ifcheckdata
,
self
.
ifManualCommit
)
tdLog
.
info
(
"start consume processor"
)
tdLog
.
info
(
"start consume processor"
)
tmqCom
.
startTmqSimProcess
(
self
.
pollDelay
,
self
.
paraDict
[
"dbName"
],
self
.
showMsg
,
self
.
showRow
,
self
.
cdbName
)
tmqCom
.
getStartConsumeNotifyFromTmqsim
()
tmqCom
.
getStartConsumeNotifyFromTmqsim
()
tdLog
.
info
(
"drop one stable"
)
self
.
paraDict
[
"stbName"
]
=
'stb1'
tdSql
.
execute
(
"drop table %s.%s"
%
(
self
.
paraDict
[
'dbName'
],
self
.
paraDict
[
'stbName'
]))
self
.
paraDict
[
"stbName"
]
=
'stb1'
tdSql
.
execute
(
"drop table %s.%s"
%
(
self
.
paraDict
[
'dbName'
],
self
.
paraDict
[
'stbName'
]))
# tmqCom.drop_ctable(tdSql, dbname=self.paraDict['dbName'], count=self.paraDict["ctbNum"], default_ctbname_prefix=self.paraDict["ctbPrefix"])
pThread2
.
join
()
tdLog
.
info
(
"wait result from consumer, then check it"
)
expectRows
=
1
resultList
=
tmqCom
.
selectConsumeResult
(
expectRows
)
...
...
@@ -105,7 +105,7 @@ class TDTestCase:
totalConsumeRows
=
0
for
i
in
range
(
expectRows
):
totalConsumeRows
+=
resultList
[
i
]
if
not
(
totalConsumeRows
>=
self
.
expectrowcnt
/
2
and
totalConsumeRows
<=
self
.
expectrowcnt
):
tdLog
.
info
(
"act consume rows: %d, expect consume rows: between %d and %d"
%
(
totalConsumeRows
,
self
.
expectrowcnt
/
2
,
self
.
expectrowcnt
))
tdLog
.
exit
(
"tmq consume rows error!"
)
...
...
tests/system-test/7-tmq/tmqError.py
浏览文件 @
b7fae566
...
...
@@ -56,7 +56,7 @@ class TDTestCase:
print
(
cur
)
return
cur
def
initConsumerTable
(
self
,
cdbName
=
'cdb'
):
def
initConsumerTable
(
self
,
cdbName
=
'cdb'
):
tdLog
.
info
(
"create consume database, and consume info table, and consume result table"
)
tdSql
.
query
(
"create database if not exists %s vgroups 1"
%
(
cdbName
))
tdSql
.
query
(
"drop table if exists %s.consumeinfo "
%
(
cdbName
))
...
...
@@ -65,12 +65,12 @@ class TDTestCase:
tdSql
.
query
(
"create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"
%
cdbName
)
tdSql
.
query
(
"create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"
%
cdbName
)
def
initConsumerInfoTable
(
self
,
cdbName
=
'cdb'
):
def
initConsumerInfoTable
(
self
,
cdbName
=
'cdb'
):
tdLog
.
info
(
"drop consumeinfo table"
)
tdSql
.
query
(
"drop table if exists %s.consumeinfo "
%
(
cdbName
))
tdSql
.
query
(
"create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"
%
cdbName
)
def
insertConsumerInfo
(
self
,
consumerId
,
expectrowcnt
,
topicList
,
keyList
,
ifcheckdata
,
ifmanualcommit
,
cdbName
=
'cdb'
):
def
insertConsumerInfo
(
self
,
consumerId
,
expectrowcnt
,
topicList
,
keyList
,
ifcheckdata
,
ifmanualcommit
,
cdbName
=
'cdb'
):
sql
=
"insert into %s.consumeinfo values "
%
cdbName
sql
+=
"(now, %d, '%s', '%s', %d, %d, %d)"
%
(
consumerId
,
topicList
,
keyList
,
expectrowcnt
,
ifcheckdata
,
ifmanualcommit
)
tdLog
.
info
(
"consume info sql: %s"
%
sql
)
...
...
@@ -85,11 +85,11 @@ class TDTestCase:
break
else
:
time
.
sleep
(
5
)
for
i
in
range
(
expectRows
):
tdLog
.
info
(
"consume id: %d, consume msgs: %d, consume rows: %d"
%
(
tdSql
.
getData
(
i
,
1
),
tdSql
.
getData
(
i
,
2
),
tdSql
.
getData
(
i
,
3
)))
resultList
.
append
(
tdSql
.
getData
(
i
,
3
))
return
resultList
def
startTmqSimProcess
(
self
,
buildPath
,
cfgPath
,
pollDelay
,
dbName
,
showMsg
=
1
,
showRow
=
1
,
cdbName
=
'cdb'
,
valgrind
=
0
):
...
...
@@ -97,14 +97,14 @@ class TDTestCase:
logFile
=
cfgPath
+
'/../log/valgrind-tmq.log'
shellCmd
=
'nohup valgrind --log-file='
+
logFile
shellCmd
+=
'--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes '
if
(
platform
.
system
().
lower
()
==
'windows'
):
shellCmd
=
'mintty -h never -w hide '
+
buildPath
+
'
\\
build
\\
bin
\\
tmq_sim.exe -c '
+
cfgPath
shellCmd
+=
" -y %d -d %s -g %d -r %d -w %s "
%
(
pollDelay
,
dbName
,
showMsg
,
showRow
,
cdbName
)
shellCmd
+=
"> nul 2>&1 &"
shellCmd
+=
" -y %d -d %s -g %d -r %d -w %s "
%
(
pollDelay
,
dbName
,
showMsg
,
showRow
,
cdbName
)
shellCmd
+=
"> nul 2>&1 &"
else
:
shellCmd
=
'nohup '
+
buildPath
+
'/build/bin/tmq_sim -c '
+
cfgPath
shellCmd
+=
" -y %d -d %s -g %d -r %d -w %s "
%
(
pollDelay
,
dbName
,
showMsg
,
showRow
,
cdbName
)
shellCmd
+=
" -y %d -d %s -g %d -r %d -w %s "
%
(
pollDelay
,
dbName
,
showMsg
,
showRow
,
cdbName
)
shellCmd
+=
"> /dev/null 2>&1 &"
tdLog
.
info
(
shellCmd
)
os
.
system
(
shellCmd
)
...
...
@@ -134,7 +134,7 @@ class TDTestCase:
sql
=
pre_create
if
sql
!=
pre_create
:
tsql
.
execute
(
sql
)
tdLog
.
debug
(
"complete to create %d child tables in %s.%s"
%
(
ctbNum
,
dbName
,
stbName
))
return
...
...
@@ -149,7 +149,7 @@ class TDTestCase:
startTs
=
int
(
round
(
t
*
1000
))
#tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
rowsOfSql
=
0
rowsOfSql
=
0
for
i
in
range
(
ctbNum
):
sql
+=
" %s_%d values "
%
(
stbName
,
i
)
for
j
in
range
(
rowsPerTbl
):
...
...
@@ -168,8 +168,8 @@ class TDTestCase:
tsql
.
execute
(
sql
)
tdLog
.
debug
(
"insert data ............ [OK]"
)
return
def
prepareEnv
(
self
,
**
parameterDict
):
def
prepareEnv
(
self
,
**
parameterDict
):
# create new connector for my thread
tsql
=
self
.
newcur
(
parameterDict
[
'cfg'
],
'localhost'
,
6030
)
...
...
@@ -189,11 +189,11 @@ class TDTestCase:
def
tmqCase1
(
self
,
cfgPath
,
buildPath
):
'''
Leave a TMQ process. Stop taosd, delete the data directory, restart taosd,
Leave a TMQ process. Stop taosd, delete the data directory, restart taosd,
and restart a consumption process to complete a consumption
'''
tdLog
.
printNoPrefix
(
"======== test case 1: "
)
self
.
initConsumerTable
()
# create and start thread
...
...
@@ -217,7 +217,7 @@ class TDTestCase:
tdLog
.
info
(
"create topics from stb1"
)
topicFromStb1
=
'topic_stb1'
tdSql
.
execute
(
"create topic %s as select ts, c1, c2 from %s.%s"
%
(
topicFromStb1
,
parameterDict
[
'dbName'
],
parameterDict
[
'stbName'
]))
consumerId
=
0
# expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
...
...
@@ -236,7 +236,7 @@ class TDTestCase:
showMsg
=
1
showRow
=
1
self
.
startTmqSimProcess
(
buildPath
,
cfgPath
,
pollDelay
,
parameterDict
[
"dbName"
],
showMsg
,
showRow
)
time
.
sleep
(
3
)
tdLog
.
info
(
"================= stop dnode, and remove data file, then start dnode ==========================="
)
tdDnodes
.
stop
(
1
)
...
...
@@ -248,7 +248,7 @@ class TDTestCase:
tdDnodes
.
start
(
1
)
time
.
sleep
(
2
)
######### redo to consume
######### redo to consume
self
.
initConsumerTable
()
self
.
create_database
(
tdSql
,
parameterDict
[
"dbName"
])
...
...
@@ -258,7 +258,7 @@ class TDTestCase:
tdLog
.
info
(
"create topics from stb1"
)
topicFromStb1
=
'topic_stb1'
tdSql
.
execute
(
"create topic %s as select ts, c1, c2 from %s.%s"
%
(
topicFromStb1
,
parameterDict
[
'dbName'
],
parameterDict
[
'stbName'
]))
consumerId
=
0
expectrowcnt
=
parameterDict
[
"rowsPerTbl"
]
*
parameterDict
[
"ctbNum"
]
...
...
@@ -282,7 +282,7 @@ class TDTestCase:
totalConsumeRows
=
0
for
i
in
range
(
expectRows
):
totalConsumeRows
+=
resultList
[
i
]
tdLog
.
info
(
"act consume rows: %d, expect consume rows: %d"
%
(
totalConsumeRows
,
expectrowcnt
))
if
not
(
totalConsumeRows
==
expectrowcnt
):
tdLog
.
exit
(
"tmq consume rows error!"
)
...
...
@@ -293,7 +293,7 @@ class TDTestCase:
else
:
os
.
system
(
'pkill tmq_sim'
)
tdLog
.
printNoPrefix
(
"======== test case 1 end ...... "
)
tdLog
.
printNoPrefix
(
"======== test case 1 end ...... "
)
def
run
(
self
):
tdSql
.
prepare
()
...
...
tests/system-test/7-tmq/tmqModule.py
浏览文件 @
b7fae566
此差异已折叠。
点击以展开。
tests/system-test/7-tmq/tmqSubscribeStb-r3.py
浏览文件 @
b7fae566
...
...
@@ -39,20 +39,20 @@ class TDTestCase:
cmdStr
=
'%s/build/bin/taos -c %s -s "%s >> %s"'
%
(
buildPath
,
cfgPath
,
queryString
,
dstFile
)
tdLog
.
info
(
cmdStr
)
os
.
system
(
cmdStr
)
consumeRowsFile
=
'%s/../log/consumerid_%d.txt'
%
(
cfgPath
,
consumerId
)
tdLog
.
info
(
"rows file: %s, %s"
%
(
consumeRowsFile
,
dstFile
))
consumeFile
=
open
(
consumeRowsFile
,
mode
=
'r'
)
queryFile
=
open
(
dstFile
,
mode
=
'r'
)
# skip first line for it is schema
queryFile
.
readline
()
while
True
:
dst
=
queryFile
.
readline
()
src
=
consumeFile
.
readline
()
if
dst
:
if
dst
!=
src
:
tdLog
.
exit
(
"consumerId %d consume rows is not match the rows by direct query"
%
consumerId
)
...
...
@@ -85,7 +85,7 @@ class TDTestCase:
paraDict
[
'vgroups'
]
=
self
.
vgroups
paraDict
[
'ctbNum'
]
=
self
.
ctbNum
paraDict
[
'rowsPerTbl'
]
=
self
.
rowsPerTbl
tmqCom
.
initConsumerTable
()
tdCom
.
create_database
(
tdSql
,
paraDict
[
"dbName"
],
paraDict
[
"dropFlag"
],
vgroups
=
paraDict
[
"vgroups"
],
replica
=
self
.
replica
)
tdLog
.
info
(
"create stb"
)
...
...
@@ -101,7 +101,7 @@ class TDTestCase:
# ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
# startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
tmqCom
.
asyncInsertDataByInterlace
(
paraDict
)
tdLog
.
info
(
"wait some data inserted"
)
exitFlag
=
1
while
exitFlag
:
...
...
@@ -112,7 +112,7 @@ class TDTestCase:
if
(
rowsInserted
>
((
self
.
ctbNum
*
self
.
rowsPerTbl
)
/
5
)):
exitFlag
=
0
time
.
sleep
(
0.1
)
tdLog
.
info
(
"inserted rows: %d"
%
tdSql
.
getData
(
0
,
0
))
# tdDnodes=cluster.dnodes
tdLog
.
info
(
"================= restart dnode 2==========================="
)
...
...
@@ -123,18 +123,18 @@ class TDTestCase:
cluster
.
dnodes
[
2
].
starttaosd
()
tdLog
.
info
(
"================= restart dnode 4==========================="
)
cluster
.
dnodes
[
3
].
stoptaosd
()
cluster
.
dnodes
[
3
].
starttaosd
()
cluster
.
dnodes
[
3
].
starttaosd
()
tdLog
.
info
(
"================= restart dnode 5==========================="
)
cluster
.
dnodes
[
4
].
stoptaosd
()
cluster
.
dnodes
[
4
].
starttaosd
()
# tdLog.info("restart taosd to ensure that the data falls into the disk")
# tdLog.info("restart taosd to ensure that the data falls into the disk")
# tdSql.query("flush database %s"%(paraDict['dbName']))
return
def
tmqCase1
(
self
):
tdLog
.
printNoPrefix
(
"======== test case 1: "
)
tdLog
.
printNoPrefix
(
"======== test case 1: "
)
# create and start thread
paraDict
=
{
'dbName'
:
'dbt'
,
'dropFlag'
:
1
,
...
...
@@ -159,14 +159,14 @@ class TDTestCase:
paraDict
[
'vgroups'
]
=
self
.
vgroups
paraDict
[
'ctbNum'
]
=
self
.
ctbNum
paraDict
[
'rowsPerTbl'
]
=
self
.
rowsPerTbl
tdLog
.
info
(
"create topics from stb1"
)
topicFromStb1
=
'topic_stb1'
topicFromStb1
=
'topic_stb1'
queryString
=
"select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha' "
%
(
paraDict
[
'dbName'
],
paraDict
[
'stbName'
])
sqlString
=
"create topic %s as %s"
%
(
topicFromStb1
,
queryString
)
tdLog
.
info
(
"create topic sql: %s"
%
sqlString
)
tdSql
.
execute
(
sqlString
)
consumerId
=
0
expectrowcnt
=
paraDict
[
"rowsPerTbl"
]
*
paraDict
[
"ctbNum"
]
topicList
=
topicFromStb1
...
...
@@ -178,9 +178,9 @@ class TDTestCase:
auto.offset.reset:earliest'
tmqCom
.
insertConsumerInfo
(
consumerId
,
expectrowcnt
,
topicList
,
keyList
,
ifcheckdata
,
ifManualCommit
)
tdLog
.
info
(
"start consume processor"
)
tdLog
.
info
(
"start consume processor"
)
tmqCom
.
startTmqSimProcess
(
pollDelay
=
paraDict
[
'pollDelay'
],
dbName
=
paraDict
[
"dbName"
],
showMsg
=
paraDict
[
'showMsg'
],
showRow
=
paraDict
[
'showRow'
],
snapshot
=
paraDict
[
'snapshot'
])
tdLog
.
info
(
"start to check consume result"
)
expectRows
=
1
resultList
=
tmqCom
.
selectConsumeResult
(
expectRows
)
...
...
@@ -190,13 +190,13 @@ class TDTestCase:
tdSql
.
query
(
queryString
)
totalRowsFromQuery
=
tdSql
.
getRows
()
tdLog
.
info
(
"act consume rows: %d, act query rows: %d, expect consume rows: %d, "
%
(
totalConsumeRows
,
totalRowsFromQuery
,
expectrowcnt
))
if
totalConsumeRows
!=
totalRowsFromQuery
:
tdLog
.
exit
(
"tmq consume rows error!"
)
# tmqCom.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
tmqCom
.
waitSubscriptionExit
(
tdSql
,
topicFromStb1
)
tdSql
.
query
(
"drop topic %s"
%
topicFromStb1
)
...
...
@@ -204,8 +204,8 @@ class TDTestCase:
tdLog
.
printNoPrefix
(
"======== test case 1 end ...... "
)
def
tmqCase2
(
self
):
tdLog
.
printNoPrefix
(
"======== test case 2: "
)
tdLog
.
printNoPrefix
(
"======== test case 2: "
)
# create and start thread
paraDict
=
{
'dbName'
:
'dbt'
,
'dropFlag'
:
1
,
...
...
@@ -230,14 +230,14 @@ class TDTestCase:
paraDict
[
'vgroups'
]
=
self
.
vgroups
paraDict
[
'ctbNum'
]
=
self
.
ctbNum
paraDict
[
'rowsPerTbl'
]
=
self
.
rowsPerTbl
tdLog
.
info
(
"create topics from stb1"
)
topicFromStb1
=
'topic_stb1'
topicFromStb1
=
'topic_stb1'
queryString
=
"select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha' "
%
(
paraDict
[
'dbName'
],
paraDict
[
'stbName'
])
sqlString
=
"create topic %s as %s"
%
(
topicFromStb1
,
queryString
)
tdLog
.
info
(
"create topic sql: %s"
%
sqlString
)
tdSql
.
execute
(
sqlString
)
consumerId
=
0
expectrowcnt
=
paraDict
[
"rowsPerTbl"
]
*
paraDict
[
"ctbNum"
]
topicList
=
topicFromStb1
...
...
@@ -249,7 +249,7 @@ class TDTestCase:
auto.offset.reset:earliest'
tmqCom
.
insertConsumerInfo
(
consumerId
,
expectrowcnt
,
topicList
,
keyList
,
ifcheckdata
,
ifManualCommit
)
tdLog
.
info
(
"start consume processor"
)
tdLog
.
info
(
"start consume processor"
)
tmqCom
.
startTmqSimProcess
(
pollDelay
=
paraDict
[
'pollDelay'
],
dbName
=
paraDict
[
"dbName"
],
showMsg
=
paraDict
[
'showMsg'
],
showRow
=
paraDict
[
'showRow'
],
snapshot
=
paraDict
[
'snapshot'
])
tdLog
.
info
(
"================= restart dnode 2==========================="
)
...
...
@@ -260,11 +260,11 @@ class TDTestCase:
cluster
.
dnodes
[
2
].
starttaosd
()
tdLog
.
info
(
"================= restart dnode 4==========================="
)
cluster
.
dnodes
[
3
].
stoptaosd
()
cluster
.
dnodes
[
3
].
starttaosd
()
cluster
.
dnodes
[
3
].
starttaosd
()
tdLog
.
info
(
"================= restart dnode 5==========================="
)
cluster
.
dnodes
[
4
].
stoptaosd
()
cluster
.
dnodes
[
4
].
starttaosd
()
tdLog
.
info
(
"start to check consume result"
)
expectRows
=
1
resultList
=
tmqCom
.
selectConsumeResult
(
expectRows
)
...
...
@@ -274,13 +274,13 @@ class TDTestCase:
tdSql
.
query
(
queryString
)
totalRowsFromQuery
=
tdSql
.
getRows
()
tdLog
.
info
(
"act consume rows: %d, act query rows: %d, expect consume rows: %d, "
%
(
totalConsumeRows
,
totalRowsFromQuery
,
expectrowcnt
))
if
totalConsumeRows
!=
totalRowsFromQuery
:
tdLog
.
exit
(
"tmq consume rows error!"
)
# tmqCom.checkFileContent(consumerId, queryString)
# tmqCom.checkFileContent(consumerId, queryString)
tmqCom
.
waitSubscriptionExit
(
tdSql
,
topicFromStb1
)
tdSql
.
query
(
"drop topic %s"
%
topicFromStb1
)
...
...
tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py
浏览文件 @
b7fae566
...
...
@@ -22,12 +22,12 @@ class TDTestCase:
self
.
vgroups
=
4
self
.
ctbNum
=
100
self
.
rowsPerTbl
=
1000
def
init
(
self
,
conn
,
logSql
):
tdLog
.
debug
(
f
"start to excute
{
__file__
}
"
)
tdSql
.
init
(
conn
.
cursor
())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
def
prepare_udf_so
(
self
):
selfPath
=
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
))
...
...
@@ -66,20 +66,20 @@ class TDTestCase:
cmdStr
=
'%s/build/bin/taos -c %s -s "%s >> %s"'
%
(
buildPath
,
cfgPath
,
queryString
,
dstFile
)
tdLog
.
info
(
cmdStr
)
os
.
system
(
cmdStr
)
consumeRowsFile
=
'%s/../log/consumerid_%d.txt'
%
(
cfgPath
,
consumerId
)
tdLog
.
info
(
"rows file: %s, %s"
%
(
consumeRowsFile
,
dstFile
))
consumeFile
=
open
(
consumeRowsFile
,
mode
=
'r'
)
queryFile
=
open
(
dstFile
,
mode
=
'r'
)
# skip first line for it is schema
queryFile
.
readline
()
while
True
:
dst
=
queryFile
.
readline
()
src
=
consumeFile
.
readline
()
if
dst
:
if
dst
!=
src
:
tdLog
.
exit
(
"consumerId %d consume rows is not match the rows by direct query"
%
consumerId
)
...
...
@@ -112,7 +112,7 @@ class TDTestCase:
paraDict
[
'vgroups'
]
=
self
.
vgroups
paraDict
[
'ctbNum'
]
=
self
.
ctbNum
paraDict
[
'rowsPerTbl'
]
=
self
.
rowsPerTbl
tmqCom
.
initConsumerTable
()
tdCom
.
create_database
(
tdSql
,
paraDict
[
"dbName"
],
paraDict
[
"dropFlag"
],
vgroups
=
paraDict
[
"vgroups"
],
replica
=
1
)
tdLog
.
info
(
"create stb"
)
...
...
@@ -127,11 +127,11 @@ class TDTestCase:
# tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx",
# ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
# startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
# tdLog.info("restart taosd to ensure that the data falls into the disk")
# tdLog.info("restart taosd to ensure that the data falls into the disk")
# tdSql.query("flush database %s"%(paraDict['dbName']))
return
def
tmqCase1
(
self
):
tdLog
.
printNoPrefix
(
"======== test case 1: multi sub table"
)
paraDict
=
{
'dbName'
:
'dbt'
,
...
...
@@ -168,13 +168,13 @@ class TDTestCase:
# tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
# tdLog.info("insert data")
# tmqCom.insert_data_1(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
tdLog
.
info
(
"create topics from stb with filter"
)
queryString
=
"select ts,c1,udf1(c1),c2,udf1(c2) from %s.%s where c1 %% 7 == 0"
%
(
paraDict
[
'dbName'
],
paraDict
[
'stbName'
])
sqlString
=
"create topic %s as %s"
%
(
topicNameList
[
0
],
queryString
)
tdLog
.
info
(
"create topic sql: %s"
%
sqlString
)
tdSql
.
execute
(
sqlString
)
tdSql
.
query
(
queryString
)
tdSql
.
query
(
queryString
)
expectRowsList
.
append
(
tdSql
.
getRows
())
# init consume info, and start tmq_sim, then check consume result
...
...
@@ -190,10 +190,10 @@ class TDTestCase:
tdLog
.
info
(
"start consume processor"
)
tmqCom
.
startTmqSimProcess
(
paraDict
[
'pollDelay'
],
paraDict
[
"dbName"
],
paraDict
[
'showMsg'
],
paraDict
[
'showRow'
],
snapshot
=
paraDict
[
'snapshot'
])
tdLog
.
info
(
"wait the consume result"
)
tdLog
.
info
(
"wait the consume result"
)
expectRows
=
1
resultList
=
tmqCom
.
selectConsumeResult
(
expectRows
)
if
expectRowsList
[
0
]
!=
resultList
[
0
]:
tdLog
.
info
(
"expect consume rows: %d, act consume rows: %d"
%
(
expectRowsList
[
0
],
resultList
[
0
]))
tdLog
.
exit
(
"0 tmq consume rows error!"
)
...
...
@@ -208,7 +208,7 @@ class TDTestCase:
sqlString
=
"create topic %s as %s"
%
(
topicNameList
[
1
],
queryString
)
tdLog
.
info
(
"create topic sql: %s"
%
sqlString
)
tdSql
.
execute
(
sqlString
)
tdSql
.
query
(
queryString
)
tdSql
.
query
(
queryString
)
expectRowsList
.
append
(
tdSql
.
getRows
())
consumerId
=
1
...
...
@@ -218,7 +218,7 @@ class TDTestCase:
tdLog
.
info
(
"start consume processor"
)
tmqCom
.
startTmqSimProcess
(
paraDict
[
'pollDelay'
],
paraDict
[
"dbName"
],
paraDict
[
'showMsg'
],
paraDict
[
'showRow'
],
snapshot
=
paraDict
[
'snapshot'
])
tdLog
.
info
(
"wait the consume result"
)
tdLog
.
info
(
"wait the consume result"
)
expectRows
=
1
resultList
=
tmqCom
.
selectConsumeResult
(
expectRows
)
if
expectRowsList
[
1
]
!=
resultList
[
0
]:
...
...
@@ -228,12 +228,12 @@ class TDTestCase:
# self.checkFileContent(consumerId, queryString)
# tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
time
.
sleep
(
10
)
time
.
sleep
(
10
)
for
i
in
range
(
len
(
topicNameList
)):
tdSql
.
query
(
"drop topic %s"
%
topicNameList
[
i
])
tdLog
.
printNoPrefix
(
"======== test case 1 end ...... "
)
def
tmqCase2
(
self
):
tdLog
.
printNoPrefix
(
"======== test case 2: multi sub table, consume with auto create tble and insert data"
)
paraDict
=
{
'dbName'
:
'dbt'
,
...
...
@@ -270,13 +270,13 @@ class TDTestCase:
# tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
# tdLog.info("insert data")
# tmqCom.insert_data_1(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
tdLog
.
info
(
"create topics from stb with filter"
)
queryString
=
"select ts,c1,udf1(c1),c2,udf1(c2) from %s.%s where c1 %% 7 == 0"
%
(
paraDict
[
'dbName'
],
paraDict
[
'stbName'
])
sqlString
=
"create topic %s as %s"
%
(
topicNameList
[
0
],
queryString
)
tdLog
.
info
(
"create topic sql: %s"
%
sqlString
)
tdSql
.
execute
(
sqlString
)
# tdSql.query(queryString)
# tdSql.query(queryString)
# expectRowsList.append(tdSql.getRows())
# init consume info, and start tmq_sim, then check consume result
...
...
@@ -292,18 +292,18 @@ class TDTestCase:
tdLog
.
info
(
"start consume processor"
)
tmqCom
.
startTmqSimProcess
(
paraDict
[
'pollDelay'
],
paraDict
[
"dbName"
],
paraDict
[
'showMsg'
],
paraDict
[
'showRow'
],
snapshot
=
paraDict
[
'snapshot'
])
paraDict
[
'startTs'
]
=
paraDict
[
'startTs'
]
+
int
(
self
.
rowsPerTbl
)
paraDict
[
'startTs'
]
=
paraDict
[
'startTs'
]
+
int
(
self
.
rowsPerTbl
)
tmqCom
.
insert_data_interlaceByMultiTbl
(
tsql
=
tdSql
,
dbName
=
paraDict
[
"dbName"
],
ctbPrefix
=
paraDict
[
"ctbPrefix"
],
ctbNum
=
paraDict
[
"ctbNum"
],
rowsPerTbl
=
paraDict
[
"rowsPerTbl"
],
batchNum
=
paraDict
[
"batchNum"
],
startTs
=
paraDict
[
"startTs"
],
ctbStartIdx
=
paraDict
[
'ctbStartIdx'
])
tdLog
.
info
(
"wait the consume result"
)
tdLog
.
info
(
"wait the consume result"
)
expectRows
=
1
resultList
=
tmqCom
.
selectConsumeResult
(
expectRows
)
tdSql
.
query
(
queryString
)
tdSql
.
query
(
queryString
)
expectRowsList
.
append
(
tdSql
.
getRows
())
if
expectRowsList
[
0
]
!=
resultList
[
0
]:
tdLog
.
info
(
"expect consume rows: %d, act consume rows: %d"
%
(
expectRowsList
[
0
],
resultList
[
0
]))
tdLog
.
exit
(
"2 tmq consume rows error!"
)
...
...
@@ -318,7 +318,7 @@ class TDTestCase:
sqlString
=
"create topic %s as %s"
%
(
topicNameList
[
1
],
queryString
)
tdLog
.
info
(
"create topic sql: %s"
%
sqlString
)
tdSql
.
execute
(
sqlString
)
tdSql
.
query
(
queryString
)
tdSql
.
query
(
queryString
)
expectRowsList
.
append
(
tdSql
.
getRows
())
consumerId
=
3
...
...
@@ -328,7 +328,7 @@ class TDTestCase:
tdLog
.
info
(
"start consume processor"
)
tmqCom
.
startTmqSimProcess
(
paraDict
[
'pollDelay'
],
paraDict
[
"dbName"
],
paraDict
[
'showMsg'
],
paraDict
[
'showRow'
],
snapshot
=
paraDict
[
'snapshot'
])
tdLog
.
info
(
"wait the consume result"
)
tdLog
.
info
(
"wait the consume result"
)
expectRows
=
1
resultList
=
tmqCom
.
selectConsumeResult
(
expectRows
)
if
expectRowsList
[
1
]
!=
resultList
[
0
]:
...
...
@@ -338,7 +338,7 @@ class TDTestCase:
# self.checkFileContent(consumerId, queryString)
# tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId))
time
.
sleep
(
10
)
time
.
sleep
(
10
)
for
i
in
range
(
len
(
topicNameList
)):
tdSql
.
query
(
"drop topic %s"
%
topicNameList
[
i
])
...
...
@@ -348,13 +348,13 @@ class TDTestCase:
# tdSql.prepare()
self
.
prepare_udf_so
()
self
.
create_udf_function
()
tdLog
.
printNoPrefix
(
"============================================="
)
tdLog
.
printNoPrefix
(
"======== snapshot is 0: only consume from wal"
)
self
.
prepareTestEnv
()
self
.
tmqCase1
()
self
.
tmqCase2
()
# tdLog.printNoPrefix("====================================================================")
# tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal")
# self.prepareTestEnv()
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录