diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a03264549306df55e63ab87dcd7fe5b294cfbffc..6100d416b11d22df45cca1119c1ebfad50891095 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -658,6 +658,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWin void cleanupAggSup(SAggSupporter* pAggSup); void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); +void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId); SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode); SColumn extractColumnFromColumnNode(SColumnNode* pColNode); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0ab172f03a3f0cfd07eedcd37e5336e43701057d..1cfabf2975c2805883ed9af802109cb9d5ed4442 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2055,6 +2055,11 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); + // it is a reserved column for scalar function, and no data in this column yet. + if (pSrc->pData == NULL) { + continue; + } + int32_t numOfRows = 0; for (int32_t j = 0; j < totalRows; ++j) { if (rowRes[j] == 0) { @@ -4372,9 +4377,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p goto _error; } - pInfo->limit = *pLimit; - pInfo->slimit = *pSlimit; - pInfo->curOffset = pLimit->offset; + pInfo->limit = *pLimit; + pInfo->slimit = *pSlimit; + pInfo->curOffset = pLimit->offset; pInfo->curSOffset = pSlimit->offset; pInfo->binfo.pRes = pResBlock; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 65bd8f4bda6384ccd92171c586f36a5b4ba1fda0..08539206a6271e54b4c3adaeda6a86dd6fd73ae5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -291,20 +291,7 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) // this is to handle the tbname if (fmIsScanPseudoColumnFunc(functionId)) { - struct SScalarFuncExecFuncs fpSet = {0}; - fmGetScalarFuncExecFuncs(functionId, &fpSet); - - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_BIGINT; - infoData.info.bytes = sizeof(uint64_t); - colInfoDataEnsureCapacity(&infoData, 0, 1); - - colDataAppendInt64(&infoData, 0, &pBlock->info.uid); - SScalarParam srcParam = { - .numOfRows = pBlock->info.rows, .param = pTableScanInfo->readHandle.meta, .columnData = &infoData}; - - SScalarParam param = {.columnData = pColInfoData}; - fpSet.process(&srcParam, 1, ¶m); + setTbNameColData(pTableScanInfo->readHandle.meta, pBlock, pColInfoData, functionId); } else { // these are tags const char* p = metaGetTableTagVal(&mr.me, pExpr->base.pParam[0].pCol->colId); for (int32_t i = 0; i < pBlock->info.rows; ++i) { @@ -316,6 +303,23 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) metaReaderClear(&mr); } +void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId) { + struct SScalarFuncExecFuncs fpSet = {0}; + fmGetScalarFuncExecFuncs(functionId, &fpSet); + + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_BIGINT; + infoData.info.bytes = sizeof(uint64_t); + colInfoDataEnsureCapacity(&infoData, 0, 1); + + colDataAppendInt64(&infoData, 0, (int64_t*) &pBlock->info.uid); + SScalarParam srcParam = { + .numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData}; + + SScalarParam param = {.columnData = pColInfoData}; + fpSet.process(&srcParam, 1, ¶m); +} + static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SSDataBlock* pBlock = pTableScanInfo->pResBlock; diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index 0c96635a788ba18ab391cfad869e2a68e675954c..ee9e87cf047ae35cf771d1c92dc89b737bb584de 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -1,278 +1,288 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics -#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-1vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 3 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$cdb_index = 0 -#=============================== start consume =============================# - -print ================ test consume from stb -$loop_cnt = 0 -loop_consume_diff_topic_from_stb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_stb_column - $topicList = ' . topic_stb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_stb_all - $topicList = ' . topic_stb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_stb_function - $topicList = ' . topic_stb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_stb_end -endi - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi -if $data[0][3] != $expectmsgcnt then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_stb -loop_consume_diff_topic_from_stb_end: - -print ================ test consume from ctb -$loop_cnt = 0 -loop_consume_diff_topic_from_ctb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ctb_column - $topicList = ' . topic_ctb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ctb_all - $topicList = ' . topic_ctb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ctb_function - $topicList = ' . topic_ctb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ctb_end -endi - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfCtb then - return -1 -endi -if $data[0][3] != $totalMsgOfCtb then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ctb -loop_consume_diff_topic_from_ctb_end: - -print ================ test consume from ntb -$loop_cnt = 0 -loop_consume_diff_topic_from_ntb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ntb_column - $topicList = ' . topic_ntb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ntb_all - $topicList = ' . topic_ntb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ntb_function - $topicList = ' . topic_ntb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ntb_end -endi - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfNtb then - return -1 -endi -if $data[0][3] != $totalMsgOfNtb then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ntb -loop_consume_diff_topic_from_ntb_end: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 3 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + + +$cdb_index = 0 +#=============================== start consume =============================# + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end +endi + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic1Of2Cons.sim b/tests/script/tsim/tmq/basic1Of2Cons.sim index 957f1774f9e8dbefca393900478f9649373de8ef..9d4b0e75da7be0490167318ffb5bc21b81acadfd 100644 --- a/tests/script/tsim/tmq/basic1Of2Cons.sim +++ b/tests/script/tsim/tmq/basic1Of2Cons.sim @@ -1,372 +1,382 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics -#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-1vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 5 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$cdb_index = 0 -#=============================== start consume =============================# - -print ================ test consume from stb -$loop_cnt = 0 -loop_consume_diff_topic_from_stb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for stb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_stb_column - $topicList = ' . topic_stb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_stb_all - $topicList = ' . topic_stb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_stb_function - $topicList = ' . topic_stb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_stb_end -endi - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb -if $data[0][2] == $totalMsgOfStb then - if $data[1][2] == 0 then - goto check_ok_0 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfStb then - goto check_ok_0 - endi -endi -return -1 -check_ok_0: - -if $data[0][3] == $totalMsgOfStb then - if $data[1][3] == 0 then - goto check_ok_1 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfStb then - goto check_ok_1 - endi -endi -return -1 -check_ok_1: - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_stb -loop_consume_diff_topic_from_stb_end: - -print ================ test consume from ctb -$loop_cnt = 0 -loop_consume_diff_topic_from_ctb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ctb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ctb_column - $topicList = ' . topic_ctb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ctb_all - $topicList = ' . topic_ctb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ctb_function - $topicList = ' . topic_ctb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ctb_end -endi - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then - if $data[1][2] == 0 then - goto check_ok_2 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then - goto check_ok_2 - endi -endi -return -1 -check_ok_2: - -if $data[0][3] == $totalMsgOfCtb then - if $data[1][3] == 0 then - goto check_ok_3 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfCtb then - goto check_ok_3 - endi -endi -return -1 -check_ok_3: - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ctb -loop_consume_diff_topic_from_ctb_end: - -print ================ test consume from ntb -$loop_cnt = 0 -loop_consume_diff_topic_from_ntb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ntb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ntb_column - $topicList = ' . topic_ntb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ntb_all - $topicList = ' . topic_ntb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ntb_function - $topicList = ' . topic_ntb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ntb_end -endi - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[1][1] == 0 then - if $data[0][1] != 1 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then - if $data[1][2] == 0 then - goto check_ok_4 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then - goto check_ok_4 - endi -endi -return -1 -check_ok_4: - -if $data[0][3] == $totalMsgOfNtb then - if $data[1][3] == 0 then - goto check_ok_5 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfNtb then - goto check_ok_5 - endi -endi -return -1 -check_ok_5: - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ntb -loop_consume_diff_topic_from_ntb_end: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics +#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + +$cdb_index = 0 + +#=============================== start consume =============================# + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for stb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb +if $data[0][2] == $totalMsgOfStb then + if $data[1][2] == 0 then + goto check_ok_0 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfStb then + goto check_ok_0 + endi +endi +return -1 +check_ok_0: + +if $data[0][3] == $totalMsgOfStb then + if $data[1][3] == 0 then + goto check_ok_1 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfStb then + goto check_ok_1 + endi +endi +return -1 +check_ok_1: + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ctb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end +endi + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_2 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_2 + endi +endi +return -1 +check_ok_2: + +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_3 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_3 + endi +endi +return -1 +check_ok_3: + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ntb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[1][1] == 0 then + if $data[0][1] != 1 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_4 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_4 + endi +endi +return -1 +check_ok_4: + +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_5 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_5 + endi +endi +return -1 +check_ok_5: + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic2.sim b/tests/script/tsim/tmq/basic2.sim index 53f10e22475cc067a25b86b7aa0c46c342d0111c..dce73be592e459264916f67b4390141115c05bc8 100644 --- a/tests/script/tsim/tmq/basic2.sim +++ b/tests/script/tsim/tmq/basic2.sim @@ -1,219 +1,229 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics -#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-1vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 3 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$topicNum = 3 - -#=============================== start consume =============================# - - -print ================ test consume from stb -print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function -$topicList = ' . topic_stb_column -$topicList = $topicList . , -$topicList = $topicList . topic_stb_all -$topicList = $topicList . , -$topicList = $topicList . topic_stb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi -if $data[0][3] != $expectmsgcnt then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ctb -print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function -$topicList = ' . topic_ctb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfCtb then - return -1 -endi -if $data[0][3] != $totalMsgOfCtb then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ntb -print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function -$topicList = ' . topic_ntb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfNtb then - return -1 -endi -if $data[0][3] != $totalMsgOfNtb then - return -1 -endi - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 3 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + + +$topicNum = 3 + +#=============================== start consume =============================# + + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic2Of2Cons.sim b/tests/script/tsim/tmq/basic2Of2Cons.sim index 01ccb2b5154ee8f865265432ff977607c009de20..0494ddb5b826fa346d9460fa28e0b8f9133dd986 100644 --- a/tests/script/tsim/tmq/basic2Of2Cons.sim +++ b/tests/script/tsim/tmq/basic2Of2Cons.sim @@ -1,309 +1,318 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics -#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-1vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 5 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$topicNum = 3 - -#=============================== start consume =============================# - - -print ================ test consume from stb -print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function -$topicList = ' . topic_stb_column -$topicList = $topicList . , -$topicList = $topicList . topic_stb_all -$topicList = $topicList . , -$topicList = $topicList . topic_stb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb -if $data[0][2] == $totalMsgOfStb then - if $data[1][2] == 0 then - goto check_ok_0 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfStb then - goto check_ok_0 - endi -endi -return -1 -check_ok_0: - -if $data[0][3] == $totalMsgOfStb then - if $data[1][3] == 0 then - goto check_ok_1 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfStb then - goto check_ok_1 - endi -endi -return -1 -check_ok_1: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ctb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ctb -print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function -$topicList = ' . topic_ctb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then - if $data[1][2] == 0 then - goto check_ok_2 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then - goto check_ok_2 - endi -endi -return -1 -check_ok_2: - -if $data[0][3] == $totalMsgOfCtb then - if $data[1][3] == 0 then - goto check_ok_3 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfCtb then - goto check_ok_3 - endi -endi -return -1 -check_ok_3: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ntb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ntb -print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function -$topicList = ' . topic_ntb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[1][1] == 0 then - if $data[0][1] != 1 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then - if $data[1][2] == 0 then - goto check_ok_4 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then - goto check_ok_4 - endi -endi -return -1 -check_ok_4: - -if $data[0][3] == $totalMsgOfNtb then - if $data[1][3] == 0 then - goto check_ok_5 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfNtb then - goto check_ok_5 - endi -endi -return -1 -check_ok_5: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics +#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + +$topicNum = 3 + +#=============================== start consume =============================# + + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb +if $data[0][2] == $totalMsgOfStb then + if $data[1][2] == 0 then + goto check_ok_0 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfStb then + goto check_ok_0 + endi +endi +return -1 +check_ok_0: + +if $data[0][3] == $totalMsgOfStb then + if $data[1][3] == 0 then + goto check_ok_1 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfStb then + goto check_ok_1 + endi +endi +return -1 +check_ok_1: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ctb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_2 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_2 + endi +endi +return -1 +check_ok_2: + +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_3 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_3 + endi +endi +return -1 +check_ok_3: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ntb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[1][1] == 0 then + if $data[0][1] != 1 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_4 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_4 + endi +endi +return -1 +check_ok_4: + +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_5 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_5 + endi +endi +return -1 +check_ok_5: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index d5c800b0e9fbaa97b08ac3198fdd6a7892336cb4..480cf520d9c942a11c70395bd347540da773ad39 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -27,6 +27,7 @@ $tstart = 1640966400000 # 2022-01-01 00:00:00.000 $pullDelay = 5 $ifcheckdata = 1 +$ifmanualcommit = 1 $showMsg = 1 $showRow = 0 @@ -53,8 +54,16 @@ sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 # return -1 #endi +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' $keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest $keyList = $keyList . ' +print ========== key list: $keyList $topicNum = 2 @@ -72,7 +81,7 @@ $consumerId = 0 $totalMsgOfOneTopic = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfOneTopic * $topicNum $expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_stb_all @@ -80,7 +89,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_stb_function $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -158,7 +167,7 @@ sleep 500 sql use $cdbName print == create consume info table and consume result table for ctb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) sql show tables @@ -179,14 +188,14 @@ $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum $expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_ctb_function $topicList = $topicList . , $topicList = $topicList . topic_ctb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -249,7 +258,7 @@ sleep 500 sql use $cdbName print == create consume info table and consume result table for ntb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) sql show tables @@ -270,7 +279,7 @@ $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum $expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_ntb_function @@ -278,7 +287,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_ntb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start diff --git a/tests/script/tsim/tmq/basic3.sim b/tests/script/tsim/tmq/basic3.sim index de771ba892ec01825fbed08464a7d3794a5226bf..8d677766d74371209e53bab94b62a524318146ee 100644 --- a/tests/script/tsim/tmq/basic3.sim +++ b/tests/script/tsim/tmq/basic3.sim @@ -1,278 +1,288 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics -#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-4vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 4 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 3 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$cdb_index = 0 -#=============================== start consume =============================# - -print ================ test consume from stb -$loop_cnt = 0 -loop_consume_diff_topic_from_stb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_stb_column - $topicList = ' . topic_stb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_stb_all - $topicList = ' . topic_stb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_stb_function - $topicList = ' . topic_stb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_stb_end -endi - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi -if $data[0][3] != $expectmsgcnt then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_stb -loop_consume_diff_topic_from_stb_end: - -print ================ test consume from ctb -$loop_cnt = 0 -loop_consume_diff_topic_from_ctb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ctb_column - $topicList = ' . topic_ctb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ctb_all - $topicList = ' . topic_ctb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ctb_function - $topicList = ' . topic_ctb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ctb_end -endi - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfCtb then - return -1 -endi -if $data[0][3] != $totalMsgOfCtb then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ctb -loop_consume_diff_topic_from_ctb_end: - -print ================ test consume from ntb -$loop_cnt = 0 -loop_consume_diff_topic_from_ntb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ntb_column - $topicList = ' . topic_ntb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ntb_all - $topicList = ' . topic_ntb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ntb_function - $topicList = ' . topic_ntb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ntb_end -endi - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfNtb then - return -1 -endi -if $data[0][3] != $totalMsgOfNtb then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ntb -loop_consume_diff_topic_from_ntb_end: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 3 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + + +$cdb_index = 0 +#=============================== start consume =============================# + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end +endi + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic3Of2Cons.sim b/tests/script/tsim/tmq/basic3Of2Cons.sim index bf640ae1a1be5d710b6fbe588ff0b99662a3f5f2..afaf824acbfd914531af27ac19257e6f6844efa7 100644 --- a/tests/script/tsim/tmq/basic3Of2Cons.sim +++ b/tests/script/tsim/tmq/basic3Of2Cons.sim @@ -1,384 +1,393 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics -#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-4vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 4 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 5 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$cdb_index = 0 -#=============================== start consume =============================# - -print ================ test consume from stb -$loop_cnt = 0 -loop_consume_diff_topic_from_stb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_stb_column - $topicList = ' . topic_stb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_stb_all - $topicList = ' . topic_stb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_stb_function - $topicList = ' . topic_stb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_stb_end -endi - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -if $data[0][2] <= 0 then - return -1 -endi -if $data[0][2] >= $expectmsgcnt then - return -1 -endi - -if $data[1][2] <= 0 then - return -1 -endi -if $data[1][2] >= $expectmsgcnt then - return -1 -endi - -$sumOfMsgCnt = $data[0][2] + $data[1][2] -if $sumOfMsgCnt != $expectmsgcnt then - return -1 -endi - - -if $data[0][3] <= 0 then - return -1 -endi -if $data[0][3] >= $expectmsgcnt then - return -1 -endi - -if $data[1][3] <= 0 then - return -1 -endi -if $data[1][3] >= $expectmsgcnt then - return -1 -endi - -$sumOfMsgRows = $data[0][3] + $data[1][3] -if $sumOfMsgRows != $expectmsgcnt then - return -1 -endi - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_stb -loop_consume_diff_topic_from_stb_end: - -print ================ test consume from ctb -$loop_cnt = 0 -loop_consume_diff_topic_from_ctb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ctb_column - $topicList = ' . topic_ctb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ctb_all - $topicList = ' . topic_ctb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ctb_function - $topicList = ' . topic_ctb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ctb_end -endi - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then - if $data[1][2] == 0 then - goto check_ok_0 - endi -elif $data[1][2] == $totalMsgOfCtb then - if $data[0][2] == 0 then - goto check_ok_0 - endi -endi -return -1 -check_ok_0: - -if $data[0][3] == $totalMsgOfCtb then - if $data[1][3] == 0 then - goto check_ok_1 - endi -elif $data[1][3] == $totalMsgOfCtb then - if $data[0][3] == 0 then - goto check_ok_1 - endi -endi -return -1 -check_ok_1: - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ctb -loop_consume_diff_topic_from_ctb_end: - -print ================ test consume from ntb -$loop_cnt = 0 -loop_consume_diff_topic_from_ntb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ntb_column - $topicList = ' . topic_ntb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ntb_all - $topicList = ' . topic_ntb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ntb_function - $topicList = ' . topic_ntb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ntb_end -endi - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[1][1] == 0 then - if $data[0][1] != 1 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then - if $data[1][2] == 0 then - goto check_ok_2 - endi -elif $data[1][2] == $totalMsgOfNtb then - if $data[0][2] == 0 then - goto check_ok_2 - endi -endi -return -1 -check_ok_2: - -if $data[0][3] == $totalMsgOfNtb then - if $data[1][3] == 0 then - goto check_ok_3 - endi -elif $data[1][3] == $totalMsgOfNtb then - if $data[0][3] == 0 then - goto check_ok_3 - endi -endi -return -1 -check_ok_3: - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ntb -loop_consume_diff_topic_from_ntb_end: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics +#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + +$cdb_index = 0 +#=============================== start consume =============================# + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +if $data[0][2] <= 0 then + return -1 +endi +if $data[0][2] >= $expectmsgcnt then + return -1 +endi + +if $data[1][2] <= 0 then + return -1 +endi +if $data[1][2] >= $expectmsgcnt then + return -1 +endi + +$sumOfMsgCnt = $data[0][2] + $data[1][2] +if $sumOfMsgCnt != $expectmsgcnt then + return -1 +endi + + +if $data[0][3] <= 0 then + return -1 +endi +if $data[0][3] >= $expectmsgcnt then + return -1 +endi + +if $data[1][3] <= 0 then + return -1 +endi +if $data[1][3] >= $expectmsgcnt then + return -1 +endi + +$sumOfMsgRows = $data[0][3] + $data[1][3] +if $sumOfMsgRows != $expectmsgcnt then + return -1 +endi + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end +endi + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_0 + endi +elif $data[1][2] == $totalMsgOfCtb then + if $data[0][2] == 0 then + goto check_ok_0 + endi +endi +return -1 +check_ok_0: + +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_1 + endi +elif $data[1][3] == $totalMsgOfCtb then + if $data[0][3] == 0 then + goto check_ok_1 + endi +endi +return -1 +check_ok_1: + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[1][1] == 0 then + if $data[0][1] != 1 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_2 + endi +elif $data[1][2] == $totalMsgOfNtb then + if $data[0][2] == 0 then + goto check_ok_2 + endi +endi +return -1 +check_ok_2: + +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_3 + endi +elif $data[1][3] == $totalMsgOfNtb then + if $data[0][3] == 0 then + goto check_ok_3 + endi +endi +return -1 +check_ok_3: + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic4.sim b/tests/script/tsim/tmq/basic4.sim index 42023bda7e1f5d5defd0266ae5a4146f50851314..9b418f12f2b95410994cf52fd4e140a8dcc73889 100644 --- a/tests/script/tsim/tmq/basic4.sim +++ b/tests/script/tsim/tmq/basic4.sim @@ -1,216 +1,226 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics -#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-4vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 4 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 3 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$topicNum = 3 - -print ================ test consume from stb -print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function -$topicList = ' . topic_stb_column -$topicList = $topicList . , -$topicList = $topicList . topic_stb_all -$topicList = $topicList . , -$topicList = $topicList . topic_stb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi -if $data[0][3] != $expectmsgcnt then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ctb -print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function -$topicList = ' . topic_ctb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfCtb then - return -1 -endi -if $data[0][3] != $totalMsgOfCtb then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ntb -print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function -$topicList = ' . topic_ntb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfNtb then - return -1 -endi -if $data[0][3] != $totalMsgOfNtb then - return -1 -endi - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 3 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + + +$topicNum = 3 + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic4Of2Cons.sim b/tests/script/tsim/tmq/basic4Of2Cons.sim index a17d8a2f455bef554aba4117d6c2f3f4e2429e4b..510aaf0e1a43c35b47e205333462a1215e7d4e63 100644 --- a/tests/script/tsim/tmq/basic4Of2Cons.sim +++ b/tests/script/tsim/tmq/basic4Of2Cons.sim @@ -1,319 +1,328 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics -#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-4vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 4 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 5 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$topicNum = 3 - -print ================ test consume from stb -print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function -$topicList = ' . topic_stb_column -$topicList = $topicList . , -$topicList = $topicList . topic_stb_all -$topicList = $topicList . , -$topicList = $topicList . topic_stb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -if $data[0][2] <= 0 then - return -1 -endi -if $data[0][2] >= $expectmsgcnt then - return -1 -endi - -if $data[1][2] <= 0 then - return -1 -endi -if $data[1][2] >= $expectmsgcnt then - return -1 -endi - -$sumOfConsMsg = $data[0][2] + $data[1][2] -if $sumOfConsMsg != $expectmsgcnt then - return -1 -endi - -if $data[0][3] <= 0 then - return -1 -endi -if $data[0][3] >= $expectmsgcnt then - return -1 -endi - -if $data[1][3] <= 0 then - return -1 -endi -if $data[1][3] >= $expectmsgcnt then - return -1 -endi - -$sumOfConsRow = $data[0][3] + $data[1][3] -if $sumOfConsRow != $expectmsgcnt then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ctb -print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function -$topicList = ' . topic_ctb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then - if $data[1][2] == 0 then - goto check_ok_0 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then - goto check_ok_0 - endi -endi -return -1 -check_ok_0: - -if $data[0][3] == $totalMsgOfCtb then - if $data[1][3] == 0 then - goto check_ok_1 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfCtb then - goto check_ok_1 - endi -endi -return -1 -check_ok_1: - - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ntb -print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function -$topicList = ' . topic_ntb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[1][1] == 0 then - if $data[0][1] != 1 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then - if $data[1][2] == 0 then - goto check_ok_2 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then - goto check_ok_2 - endi -endi -return -1 -check_ok_2: - -if $data[0][3] == $totalMsgOfNtb then - if $data[1][3] == 0 then - goto check_ok_3 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfNtb then - goto check_ok_3 - endi -endi -return -1 -check_ok_3: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics +#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + +$topicNum = 3 + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +if $data[0][2] <= 0 then + return -1 +endi +if $data[0][2] >= $expectmsgcnt then + return -1 +endi + +if $data[1][2] <= 0 then + return -1 +endi +if $data[1][2] >= $expectmsgcnt then + return -1 +endi + +$sumOfConsMsg = $data[0][2] + $data[1][2] +if $sumOfConsMsg != $expectmsgcnt then + return -1 +endi + +if $data[0][3] <= 0 then + return -1 +endi +if $data[0][3] >= $expectmsgcnt then + return -1 +endi + +if $data[1][3] <= 0 then + return -1 +endi +if $data[1][3] >= $expectmsgcnt then + return -1 +endi + +$sumOfConsRow = $data[0][3] + $data[1][3] +if $sumOfConsRow != $expectmsgcnt then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_0 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_0 + endi +endi +return -1 +check_ok_0: + +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_1 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_1 + endi +endi +return -1 +check_ok_1: + + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[1][1] == 0 then + if $data[0][1] != 1 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_2 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_2 + endi +endi +return -1 +check_ok_2: + +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_3 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_3 + endi +endi +return -1 +check_ok_3: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim index e32a1df8025f1105ffcaee8e0f081149cb756f91..fbdea96f9387e4b204e208725f3bcb474ea6eece 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim @@ -1,88 +1,88 @@ -# stop all dnodes before start this case -system sh/stop_dnodes.sh - -# deploy dnode 1 -system sh/deploy.sh -n dnode1 -i 1 - -# add some config items for this case -#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 - -# start dnode 1 -system sh/exec.sh -n dnode1 -s start - -sql connect - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -print == create database $dbName vgroups $vgroups -sql create database $dbName vgroups $vgroups - -#wait database ready -$loop_cnt = 0 -check_db_ready: -if $loop_cnt == 10 then - print ====> database not ready! - return -1 -endi -sql show databases -print ==> rows: $rows -print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] -print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[19] != nostrict then - sleep 100 - $loop_cnt = $loop_cnt + 1 - goto check_db_ready -endi - -sql use $dbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi - -print == create super table -sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) -sql show stables -if $rows != 1 then - return -1 -endi - -print == create child table, normal table and insert data -$i = 0 -while $i < $ctbNum - $ctb = $ctbPrefix . $i - $ntb = $ntbPrefix . $i - sql create table $ctb using $stbPrefix tags( $i ) - sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) - - $x = 0 - while $x < $rowsPerCtb - $binary = ' . binary- - $binary = $binary . $i - $binary = $binary . ' - - sql insert into $ctb values ($tstart , $i , $x , $binary ) - sql insert into $ntb values ($tstart , $i , $x , $binary ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - $i = $i + 1 - $tstart = 1640966400000 -endw +# stop all dnodes before start this case +system sh/stop_dnodes.sh + +# deploy dnode 1 +system sh/deploy.sh -n dnode1 -i 1 + +# add some config items for this case +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +# start dnode 1 +system sh/exec.sh -n dnode1 -s start + +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[19] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + + $x = 0 + while $x < $rowsPerCtb + $binary = ' . binary- + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , $i , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw diff --git a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim index 4750aab214e074408f3076cdd0533315fa436ea5..8b5573486d46419ba8b500fb392406e6e31705ed 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim @@ -1,88 +1,88 @@ -# stop all dnodes before start this case -system sh/stop_dnodes.sh - -# deploy dnode 1 -system sh/deploy.sh -n dnode1 -i 1 - -# add some config items for this case -#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 - -# start dnode 1 -system sh/exec.sh -n dnode1 -s start - -sql connect - -#---- global parameters start ----# -$dbName = db -$vgroups = 4 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -print == create database $dbName vgroups $vgroups -sql create database $dbName vgroups $vgroups - -#wait database ready -$loop_cnt = 0 -check_db_ready: -if $loop_cnt == 10 then - print ====> database not ready! - return -1 -endi -sql show databases -print ==> rows: $rows -print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] -print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[19] != nostrict then - sleep 100 - $loop_cnt = $loop_cnt + 1 - goto check_db_ready -endi - -sql use $dbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi - -print == create super table -sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) -sql show stables -if $rows != 1 then - return -1 -endi - -print == create child table, normal table and insert data -$i = 0 -while $i < $ctbNum - $ctb = $ctbPrefix . $i - $ntb = $ntbPrefix . $i - sql create table $ctb using $stbPrefix tags( $i ) - sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) - - $x = 0 - while $x < $rowsPerCtb - $binary = ' . binary- - $binary = $binary . $i - $binary = $binary . ' - - sql insert into $ctb values ($tstart , $i , $x , $binary ) - sql insert into $ntb values ($tstart , $i , $x , $binary ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - $i = $i + 1 - $tstart = 1640966400000 -endw +# stop all dnodes before start this case +system sh/stop_dnodes.sh + +# deploy dnode 1 +system sh/deploy.sh -n dnode1 -i 1 + +# add some config items for this case +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +# start dnode 1 +system sh/exec.sh -n dnode1 -s start + +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[19] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + + $x = 0 + while $x < $rowsPerCtb + $binary = ' . binary- + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , $i , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 65840349bacd0f25d4eb99833c5c118ca2b523e0..c2fe25efc47064638722670304a8a0c8a869998f 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -52,7 +52,7 @@ class TDTestCase: def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) tsql.execute("use %s" %dbName) - tsql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) + tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" sql = pre_create #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) @@ -345,11 +345,11 @@ class TDTestCase: after starting consumer, create ctables ") # create and start thread parameterDict = {'cfg': '', \ - 'dbName': 'db2', \ + 'dbName': 'db3', \ 'vgroups': 1, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 10000, \ + 'rowsPerTbl': 30000, \ 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath @@ -374,22 +374,33 @@ class TDTestCase: break else: time.sleep(1) - + + tdLog.info("create stable2 for the seconde topic") + parameterDict2 = {'cfg': '', \ + 'dbName': 'db3', \ + 'vgroups': 1, \ + 'stbName': 'stb2', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 30000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict2['cfg'] = cfgPath + tdSql.execute("create stable if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(parameterDict2['dbName'], parameterDict2['stbName'])) + tdLog.info("create topics from super table") - topicFromStb = 'topic_stb_column2' - topicFromCtb = 'topic_ctb_column2' + topicFromStb = 'topic_stb_column3' + topicFromStb2 = 'topic_stb_column32' tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) - tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb2, parameterDict2['dbName'], parameterDict2['stbName'])) - time.sleep(1) tdSql.query("show topics") topic1 = tdSql.getData(0 , 0) topic2 = tdSql.getData(1 , 0) tdLog.info("show topics: %s, %s"%(topic1, topic2)) - if topic1 != topicFromStb and topic1 != topicFromCtb: + if topic1 != topicFromStb and topic1 != topicFromStb2: tdLog.exit("topic error1") - if topic2 != topicFromStb and topic2 != topicFromCtb: + if topic2 != topicFromStb and topic2 != topicFromStb2: tdLog.exit("topic error2") tdLog.info("create consume info table and consume result table") @@ -397,10 +408,9 @@ class TDTestCase: tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) - rowsOfNewCtb = 1000 consumerId = 0 - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + rowsOfNewCtb - topicList = topicFromStb + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] + topicList = topicFromStb + ',' + topicFromStb2 ifcheckdata = 0 keyList = 'group.id:cgrp1,\ enable.auto.commit:false,\ @@ -432,17 +442,13 @@ class TDTestCase: tdLog.info(shellCmd) os.system(shellCmd) - # create new child table and insert data - newCtbName = 'newctb' - tdSql.query("create table %s.%s using %s.%s tags(9999)"%(parameterDict["dbName"], newCtbName, parameterDict["dbName"], parameterDict["stbName"])) - startTs = parameterDict["startTs"] - for j in range(rowsOfNewCtb): - sql = "insert into %s.%s values (%d, %d, 'tmqrow_%d') "%(parameterDict["dbName"], newCtbName, startTs + j, j, j) - tdSql.execute(sql) - tdLog.debug("insert data into new child table ............ [OK]") + # start the second thread to create new child table and insert data + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() # wait for data ready prepareEnvThread.join() + prepareEnvThread2.join() tdLog.info("insert process end, and start to check consume result") while 1: @@ -457,7 +463,7 @@ class TDTestCase: tdSql.checkData(0 , 3, expectrowcnt) tdSql.query("drop topic %s"%topicFromStb) - tdSql.query("drop topic %s"%topicFromCtb) + tdSql.query("drop topic %s"%topicFromStb2) tdLog.printNoPrefix("======== test case 3 end ...... ") @@ -474,7 +480,7 @@ class TDTestCase: self.tmqCase1(cfgPath, buildPath) self.tmqCase2(cfgPath, buildPath) - #self.tmqCase3(cfgPath, buildPath) + self.tmqCase3(cfgPath, buildPath) def stop(self): tdSql.close() diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 1228d6174c0e1ff672cc7d6fc6cb64729b50b280..b3dba695a7a8a5943e70e056a2ed17713f2a7142 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -37,9 +37,10 @@ typedef struct { TdThread thread; int32_t consumerId; - int32_t autoCommitIntervalMs; // 1000 ms - char autoCommit[8]; // true, false - char autoOffsetRest[16]; // none, earliest, latest + int32_t ifManualCommit; + //int32_t autoCommitIntervalMs; // 1000 ms + //char autoCommit[8]; // true, false + //char autoOffsetRest[16]; // none, earliest, latest int32_t ifCheckData; int64_t expectMsgCnt; @@ -136,9 +137,9 @@ void saveConfigToLogFile() { for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); - taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); - taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); - taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); + //taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); + //taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); + //taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); taosFprintfFile(g_fp, " Topics: "); for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); @@ -232,13 +233,18 @@ static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) while (1) { TAOS_ROW row = taos_fetch_row(msg); - if (row == NULL) break; - if (0 != g_stConfInfo.showRowFlag) { - TAOS_FIELD* fields = taos_fetch_fields(msg); - int32_t numOfFields = taos_field_count(msg); - taos_print_row(buf, row, fields, numOfFields); + + if (row == NULL) break; + + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + + taos_print_row(buf, row, fields, numOfFields); + + if (0 != g_stConfInfo.showRowFlag) { taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); } + totalRows++; } @@ -316,6 +322,8 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName, pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); + taosFprintfFile(g_fp, "== save result sql: %s \n", sqlStr); + TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); @@ -384,8 +392,12 @@ void* consumeThreadFunc(void* param) { loop_consume(pInfo); - tmq_commit(pInfo->tmq, NULL, 0); - + if (pInfo->ifManualCommit) { + taosFprintfFile(g_fp, "tmq_commit() manual commit when consume end.\n"); + pPrint("tmq_commit() manual commit when consume end.\n"); + tmq_commit(pInfo->tmq, NULL, 0); + } + err = tmq_unsubscribe(pInfo->tmq); if (err) { pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); @@ -470,9 +482,9 @@ int32_t getConsumeInfo() { int32_t* lengths = taos_fetch_lengths(pRes); // set default value - g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; - memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); - memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); + //g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; + //memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); + //memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); for (int i = 0; i < num_fields; ++i) { if (row[i] == NULL || 0 == i) { @@ -489,12 +501,8 @@ int32_t getConsumeInfo() { g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); - } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { - memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, row[i], lengths[i]); - } else if ((7 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = *((int32_t*)row[i]); - } else if ((8 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { - memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, row[i], lengths[i]); + } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].ifManualCommit = *((int32_t*)row[i]); } } numOfThread++;