basic2Of2ConsOverlap.sim 12.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic2Of2ConsOverlap.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics

# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
#
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
#

run tsim/tmq/prepareBasicEnv-1vgrp.sim

#---- global parameters start ----#
$dbName    = db
$vgroups    = 1
$stbPrefix  = stb
$ctbPrefix  = ctb
$ntbPrefix  = ntb
$stbNum     = 1
$ctbNum     = 10
$ntbNum     = 10
$rowsPerCtb = 10
$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
#---- global parameters end ----#

$pullDelay    = 5
$ifcheckdata  = 1
P
plum-lihui 已提交
30
$ifmanualcommit = 1
31 32 33 34 35 36
$showMsg      = 1
$showRow      = 0

sql connect
sql use $dbName

37 38 39
print == alter database
sql alter database $dbName wal_retention_period 3600

40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb

print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0

print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0

#sql show topics
#if $rows != 9 then 
#  return -1
#endi

P
plum-lihui 已提交
60
#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
61
$keyList = ' . group.id:cgrp1
P
plum-lihui 已提交
62 63 64 65 66 67
$keyList = $keyList . ,
$keyList = $keyList . enable.auto.commit:false
#$keyList = $keyList . ,
#$keyList = $keyList . auto.commit.interval.ms:6000
#$keyList = $keyList . ,
#$keyList = $keyList . auto.offset.reset:earliest
68
$keyList = $keyList . '
P
plum-lihui 已提交
69
print ========== key list:  $keyList
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86

$topicNum = 2

#=============================== start consume =============================#


print ================ test consume from stb
print == overlap toipcs: topic_stb_column + topic_stb_all, topic_stb_function + topic_stb_all
$topicList = ' . topic_stb_column
$topicList = $topicList . ,
$topicList = $topicList . topic_stb_all
$topicList = $topicList . '

$consumerId    = 0
$totalMsgOfOneTopic = $ctbNum * $rowsPerCtb
$totalMsgOfStb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt  = $totalMsgOfStb
P
plum-lihui 已提交
87
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
88 89 90 91 92 93 94


$topicList = ' . topic_stb_all
$topicList = $topicList . ,
$topicList = $topicList . topic_stb_function
$topicList = $topicList . '
$consumerId    = 1
95
sql insert into consumeinfo values (now +1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139

print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start

print == check consume result
wait_consumer_end_from_stb:
sql select * from consumeresult
print ==> rows: $rows 
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then
  sleep 1000
  goto wait_consumer_end_from_stb
endi
if $data[0][1] == 0 then
  if $data[1][1] != 1 then
    return -1
  endi
endi
if $data[0][1] == 1 then
  if $data[1][1] != 0 then
    return -1
  endi
endi

# $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb.

if $data[0][2] < $totalMsgOfOneTopic then
  return -1
endi
if $data[0][2] > $totalMsgOfStb then
  return -1
endi
if $data[1][2] < $totalMsgOfOneTopic then
  return -1
endi
if $data[1][2] > $totalMsgOfStb then
  return -1
endi

$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb
$sumOfMsgCnt = $data[0][2] + $data[1][2]
if $sumOfMsgCnt != $totalMsgCons then
L
Liu Jicong 已提交
140 141
  print total: $totalMsgCons
  print sum: $sumOfMsgCnt
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
  return -1
endi

# $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb.
if $data[0][3] < $totalMsgOfOneTopic then
  return -1
endi
if $data[0][3] > $totalMsgOfStb then
  return -1
endi
if $data[1][3] < $totalMsgOfOneTopic then
  return -1
endi
if $data[1][3] > $totalMsgOfStb then
  return -1
endi

$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb
$sumOfRows = $data[0][3] + $data[1][3]
if $sumOfRows != $totalMsgCons then
  return -1
endi

#######################################################################################
# clear consume info and consume result 
#run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdbName = cdb1
sql create database $cdbName vgroups 1
sleep 500
sql use $cdbName

174 175 176
print == alter database
sql alter database $cdbName wal_retention_period 3600

177
print == create consume info table and consume result table for ctb
P
plum-lihui 已提交
178
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)

sql show tables
if $rows != 2 then 
  return -1
endi
#######################################################################################


print ================ test consume from ctb
print == overlap toipcs: topic_ctb_column + topic_ctb_all, topic_ctb_function + topic_ctb_all
$topicList = ' . topic_ctb_column
$topicList = $topicList . ,
$topicList = $topicList . topic_ctb_all
$topicList = $topicList . '
$consumerId    = 0

$totalMsgOfOneTopic = $rowsPerCtb
$totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt  = $totalMsgOfCtb
P
plum-lihui 已提交
199
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
200 201 202 203 204 205

$topicList = ' . topic_ctb_function
$topicList = $topicList . ,
$topicList = $topicList . topic_ctb_all
$topicList = $topicList . '
$consumerId    = 1
206
sql insert into consumeinfo values (now +1s, $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267

print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start

print == check consume result
wait_consumer_end_from_ctb:
sql select * from consumeresult
print ==> rows: $rows 
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then
  sleep 1000
  goto wait_consumer_end_from_ctb
endi
if $data[0][1] == 0 then
  if $data[1][1] != 1 then
    return -1
  endi
endi
if $data[0][1] == 1 then
  if $data[1][1] != 0 then
    return -1
  endi
endi

# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb
# or     $data[0][2] $totalMsgOfCtb      and $data[1][2] == $totalMsgOfOneTopic
if $data[0][2] == $totalMsgOfOneTopic then
  if $data[1][2] == $totalMsgOfCtb then
    goto check_ok_0
  endi
elif $data[1][2] == $totalMsgOfOneTopic then
  if $data[0][2] == $totalMsgOfCtb then
    goto check_ok_0
  endi
endi
return -1
check_ok_0:

if $data[0][3] == $totalMsgOfOneTopic then
  if $data[1][3] == $totalMsgOfCtb then
    goto check_ok_1
  endi
elif $data[1][3] == $totalMsgOfOneTopic then
  if $data[0][3] == $totalMsgOfCtb then
    goto check_ok_1
  endi
endi
return -1
check_ok_1:

#######################################################################################
# clear consume info and consume result 
#run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdbName = cdb2
sql create database $cdbName vgroups 1
sleep 500
sql use $cdbName

268 269 270
print == alter database
sql alter database $cdbName wal_retention_period 3600

271
print == create consume info table and consume result table for ntb
P
plum-lihui 已提交
272
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)

sql show tables
if $rows != 2 then 
  return -1
endi
#######################################################################################


print ================ test consume from ntb
print == overlap toipcs: topic_ntb_column + topic_ntb_all, topic_ntb_function + topic_ntb_all
$topicList = ' . topic_ntb_column
$topicList = $topicList . ,
$topicList = $topicList . topic_ntb_all
$topicList = $topicList . '

$consumerId    = 0
$totalMsgOfOneTopic = $rowsPerCtb
$totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt  = $totalMsgOfNtb
P
plum-lihui 已提交
293
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
294 295 296 297 298 299 300


$topicList = ' . topic_ntb_function
$topicList = $topicList . ,
$topicList = $topicList . topic_ntb_all
$topicList = $topicList . '
$consumerId    = 1
301
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353

print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start

print == check consume result from ntb
wait_consumer_end_from_ntb:
sql select * from consumeresult
print ==> rows: $rows 
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then
  sleep 1000
  goto wait_consumer_end_from_ntb
endi
if $data[0][1] == 0 then
  if $data[1][1] != 1 then
    return -1
  endi
endi
if $data[0][1] == 1 then
  if $data[1][1] != 0 then
    return -1
  endi
endi

# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb
# or     $data[0][2] $totalMsgOfNtb      and $data[1][2] == $totalMsgOfOneTopic
if $data[0][2] == $totalMsgOfOneTopic then
  if $data[1][2] == $totalMsgOfNtb then
    goto check_ok_2
  endi
elif $data[1][2] == $totalMsgOfOneTopic then
  if $data[0][2] == $totalMsgOfNtb then
    goto check_ok_2
  endi
endi
return -1
check_ok_2:

if $data[0][3] == $totalMsgOfOneTopic then
  if $data[1][3] == $totalMsgOfNtb then
    goto check_ok_3
  endi
elif $data[1][3] == $totalMsgOfOneTopic then
  if $data[0][3] == $totalMsgOfNtb then
    goto check_ok_3
  endi
endi
return -1
check_ok_3:

D
dapan1121 已提交
354
sql select * from performance_schema.perf_consumers
355 356 357 358
if $rows != 0 then
  return -1
endi 

D
dapan1121 已提交
359
#sql select * from performance_schema.perf_subscriptions
360 361 362 363 364 365 366 367
#if $rows != 0 then
#  return -1
#endi 

#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT

system sh/exec.sh -n dnode1 -s stop -x SIGINT