basic2Of2ConsOverlap.sim 11.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic2Of2ConsOverlap.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics

# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
#
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
#

run tsim/tmq/prepareBasicEnv-1vgrp.sim

#---- global parameters start ----#
$dbName    = db
$vgroups    = 1
$stbPrefix  = stb
$ctbPrefix  = ctb
$ntbPrefix  = ntb
$stbNum     = 1
$ctbNum     = 10
$ntbNum     = 10
$rowsPerCtb = 10
$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
#---- global parameters end ----#

$pullDelay    = 5
$ifcheckdata  = 1
P
plum-lihui 已提交
30
$ifmanualcommit = 1
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
$showMsg      = 1
$showRow      = 0

sql connect
sql use $dbName

print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb

print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0

print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0

#sql show topics
#if $rows != 9 then 
#  return -1
#endi

P
plum-lihui 已提交
57
#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
58
$keyList = ' . group.id:cgrp1
P
plum-lihui 已提交
59 60 61 62 63 64
$keyList = $keyList . ,
$keyList = $keyList . enable.auto.commit:false
#$keyList = $keyList . ,
#$keyList = $keyList . auto.commit.interval.ms:6000
#$keyList = $keyList . ,
#$keyList = $keyList . auto.offset.reset:earliest
65
$keyList = $keyList . '
P
plum-lihui 已提交
66
print ========== key list:  $keyList
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83

$topicNum = 2

#=============================== start consume =============================#


print ================ test consume from stb
print == overlap toipcs: topic_stb_column + topic_stb_all, topic_stb_function + topic_stb_all
$topicList = ' . topic_stb_column
$topicList = $topicList . ,
$topicList = $topicList . topic_stb_all
$topicList = $topicList . '

$consumerId    = 0
$totalMsgOfOneTopic = $ctbNum * $rowsPerCtb
$totalMsgOfStb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt  = $totalMsgOfStb
P
plum-lihui 已提交
84
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
85 86 87 88 89 90 91


$topicList = ' . topic_stb_all
$topicList = $topicList . ,
$topicList = $topicList . topic_stb_function
$topicList = $topicList . '
$consumerId    = 1
92
sql insert into consumeinfo values (now +1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136

print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start

print == check consume result
wait_consumer_end_from_stb:
sql select * from consumeresult
print ==> rows: $rows 
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then
  sleep 1000
  goto wait_consumer_end_from_stb
endi
if $data[0][1] == 0 then
  if $data[1][1] != 1 then
    return -1
  endi
endi
if $data[0][1] == 1 then
  if $data[1][1] != 0 then
    return -1
  endi
endi

# $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb.

if $data[0][2] < $totalMsgOfOneTopic then
  return -1
endi
if $data[0][2] > $totalMsgOfStb then
  return -1
endi
if $data[1][2] < $totalMsgOfOneTopic then
  return -1
endi
if $data[1][2] > $totalMsgOfStb then
  return -1
endi

$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb
$sumOfMsgCnt = $data[0][2] + $data[1][2]
if $sumOfMsgCnt != $totalMsgCons then
L
Liu Jicong 已提交
137 138
  print total: $totalMsgCons
  print sum: $sumOfMsgCnt
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
  return -1
endi

# $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb.
if $data[0][3] < $totalMsgOfOneTopic then
  return -1
endi
if $data[0][3] > $totalMsgOfStb then
  return -1
endi
if $data[1][3] < $totalMsgOfOneTopic then
  return -1
endi
if $data[1][3] > $totalMsgOfStb then
  return -1
endi

$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb
$sumOfRows = $data[0][3] + $data[1][3]
if $sumOfRows != $totalMsgCons then
  return -1
endi

#######################################################################################
# clear consume info and consume result 
#run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdbName = cdb1
sql create database $cdbName vgroups 1
sleep 500
sql use $cdbName

print == create consume info table and consume result table for ctb
P
plum-lihui 已提交
172
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)

sql show tables
if $rows != 2 then 
  return -1
endi
#######################################################################################


print ================ test consume from ctb
print == overlap toipcs: topic_ctb_column + topic_ctb_all, topic_ctb_function + topic_ctb_all
$topicList = ' . topic_ctb_column
$topicList = $topicList . ,
$topicList = $topicList . topic_ctb_all
$topicList = $topicList . '
$consumerId    = 0

$totalMsgOfOneTopic = $rowsPerCtb
$totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt  = $totalMsgOfCtb
P
plum-lihui 已提交
193
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
194 195 196 197 198 199

$topicList = ' . topic_ctb_function
$topicList = $topicList . ,
$topicList = $topicList . topic_ctb_all
$topicList = $topicList . '
$consumerId    = 1
200
sql insert into consumeinfo values (now +1s, $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262

print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start

print == check consume result
wait_consumer_end_from_ctb:
sql select * from consumeresult
print ==> rows: $rows 
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then
  sleep 1000
  goto wait_consumer_end_from_ctb
endi
if $data[0][1] == 0 then
  if $data[1][1] != 1 then
    return -1
  endi
endi
if $data[0][1] == 1 then
  if $data[1][1] != 0 then
    return -1
  endi
endi

# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb
# or     $data[0][2] $totalMsgOfCtb      and $data[1][2] == $totalMsgOfOneTopic
if $data[0][2] == $totalMsgOfOneTopic then
  if $data[1][2] == $totalMsgOfCtb then
    goto check_ok_0
  endi
elif $data[1][2] == $totalMsgOfOneTopic then
  if $data[0][2] == $totalMsgOfCtb then
    goto check_ok_0
  endi
endi
return -1
check_ok_0:

if $data[0][3] == $totalMsgOfOneTopic then
  if $data[1][3] == $totalMsgOfCtb then
    goto check_ok_1
  endi
elif $data[1][3] == $totalMsgOfOneTopic then
  if $data[0][3] == $totalMsgOfCtb then
    goto check_ok_1
  endi
endi
return -1
check_ok_1:

#######################################################################################
# clear consume info and consume result 
#run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdbName = cdb2
sql create database $cdbName vgroups 1
sleep 500
sql use $cdbName

print == create consume info table and consume result table for ntb
P
plum-lihui 已提交
263
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)

sql show tables
if $rows != 2 then 
  return -1
endi
#######################################################################################


print ================ test consume from ntb
print == overlap toipcs: topic_ntb_column + topic_ntb_all, topic_ntb_function + topic_ntb_all
$topicList = ' . topic_ntb_column
$topicList = $topicList . ,
$topicList = $topicList . topic_ntb_all
$topicList = $topicList . '

$consumerId    = 0
$totalMsgOfOneTopic = $rowsPerCtb
$totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt  = $totalMsgOfNtb
P
plum-lihui 已提交
284
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
285 286 287 288 289 290 291


$topicList = ' . topic_ntb_function
$topicList = $topicList . ,
$topicList = $topicList . topic_ntb_all
$topicList = $topicList . '
$consumerId    = 1
292
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344

print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start

print == check consume result from ntb
wait_consumer_end_from_ntb:
sql select * from consumeresult
print ==> rows: $rows 
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then
  sleep 1000
  goto wait_consumer_end_from_ntb
endi
if $data[0][1] == 0 then
  if $data[1][1] != 1 then
    return -1
  endi
endi
if $data[0][1] == 1 then
  if $data[1][1] != 0 then
    return -1
  endi
endi

# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb
# or     $data[0][2] $totalMsgOfNtb      and $data[1][2] == $totalMsgOfOneTopic
if $data[0][2] == $totalMsgOfOneTopic then
  if $data[1][2] == $totalMsgOfNtb then
    goto check_ok_2
  endi
elif $data[1][2] == $totalMsgOfOneTopic then
  if $data[0][2] == $totalMsgOfNtb then
    goto check_ok_2
  endi
endi
return -1
check_ok_2:

if $data[0][3] == $totalMsgOfOneTopic then
  if $data[1][3] == $totalMsgOfNtb then
    goto check_ok_3
  endi
elif $data[1][3] == $totalMsgOfOneTopic then
  if $data[0][3] == $totalMsgOfNtb then
    goto check_ok_3
  endi
endi
return -1
check_ok_3:

D
dapan1121 已提交
345
sql select * from performance_schema.perf_consumers
346 347 348 349
if $rows != 0 then
  return -1
endi 

D
dapan1121 已提交
350
#sql select * from performance_schema.perf_subscriptions
351 352 353 354 355 356 357 358
#if $rows != 0 then
#  return -1
#endi 

#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT

system sh/exec.sh -n dnode1 -s stop -x SIGINT