From 82ad11630dfce99c92986e06f3051ca17dc1a3d9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Mar 2021 15:44:06 +0800 Subject: [PATCH] Feature/sangshuduo/td 3416 taosdemo interlace testcase (#5524) * [TD-3316] : add testcase for taosdemo limit and offset. check offset 0. * [TD-3316] : add testcase for taosdemo limit and offset. fix sample file import bug. * [TD-3316] : add test case for limit and offset. fix sample data issue. * [TD-3327] : fix taosdemo segfault when import data from sample data file. * [TD-3317] : make taosdemo support interlace mode. json parameter rows_per_tbl support. * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode insertion. refactor. * [TD-3317] : support interlace mode insertion. change json file. * [TD-3317] : support interlace mode insertion. fix multithread create table regression. * [TD-3317] : support interlace mode insertion. working but not perfect. * [TD-3317] : support interlace mode insertion. rename lowaTest with taosdemoTestWithJson * [TD-3317] : support interlace mode insertion. perfect * [TD-3317] : support interlace mode insertion. cleanup. * [TD-3317] : support interlace mode insertion. adjust algorithm of loop times. * [TD-3317] : support interlace mode insertion. fix delay time bug. * [TD-3317] : support interlace mode insertion. fix progressive timestamp bug. * [TD-3317] : support interlace mode insertion. add an option for performance print. * [TD-3317] : support interlace mode insertion. change json test case with less table for acceleration. * [TD-3317] : support interlace mode insertion. change progressive mode timestamp step and testcase. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3317] : support interlace mode insertion. add time shift for no sleep time. * [TD-3317] : support interlace insert. rework timestamp. * [TD-3416] : add taosdemo interlace testcase. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 18 +++++++++++------- tests/pytest/fulltest.sh | 1 + tests/pytest/tools/taosdemoTestInterlace.py | 20 +++++++++++++++----- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 51c94a8fa1..0943e0b3a9 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4287,13 +4287,14 @@ static int generateDataTail(char *tableName, int32_t tableSeq, if ((g_args.disorderRatio != 0) && (rand_num < g_args.disorderRange)) { - int64_t d = startTime - taosRandom() % 1000000 + rand_num; + int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k + - taosRandom() % 1000000 + rand_num; len = generateData(data, data_type, ncols_per_record, d, lenOfBinary); } else { len = generateData(data, data_type, ncols_per_record, - startTime + DEFAULT_TIMESTAMP_STEP * startFrom, + startTime + DEFAULT_TIMESTAMP_STEP * k, lenOfBinary); } @@ -4402,7 +4403,8 @@ static int generateDataBuffer(char *pTblName, int k; int dataLen; k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo, - g_args.num_of_RPR, pstr, insertRows, startFrom, startTime, + g_args.num_of_RPR, pstr, insertRows, startFrom, + startTime, pSamplePos, &dataLen); return k; } @@ -4475,7 +4477,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int generatedRecPerTbl = 0; bool flagSleep = true; int sleepTimeTotal = 0; - int timeShift = 0; while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampUs(); @@ -4513,16 +4514,18 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { generateDataTail( tableName, tableSeq, pThreadInfo, superTblInfo, batchPerTbl, pstr, insertRows, 0, - startTime + timeShift + sleepTimeTotal, + startTime, &(pThreadInfo->samplePos), &dataLen); + pstr += dataLen; recOfBatch += batchPerTbl; + startTime += batchPerTbl * superTblInfo->timeStampStep; pThreadInfo->totalInsertRows += batchPerTbl; + verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", pThreadInfo->threadID, __func__, __LINE__, batchPerTbl, recOfBatch); - timeShift ++; tableSeq ++; if (insertMode == INTERLACE_INSERT_MODE) { if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { @@ -4668,13 +4671,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int generated = generateDataBuffer( tableName, tableSeq, pThreadInfo, buffer, insertRows, - i, start_time + pThreadInfo->totalInsertRows * timeStampStep, + i, start_time, &(pThreadInfo->samplePos)); if (generated > 0) i += generated; else goto free_and_statistics_2; + start_time += generated * timeStampStep; pThreadInfo->totalInsertRows += generated; startTs = taosGetTimestampUs(); diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 46a1abf12c..5037f2c399 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -238,6 +238,7 @@ python3 test.py -f tools/taosdemoTestLimitOffset.py python3 test.py -f tools/taosdumpTest.py python3 test.py -f tools/taosdemoTest2.py python3 test.py -f tools/taosdemoTestSampleData.py +python3 test.py -f tools/taosdemoTestInterlace.py # subscribe python3 test.py -f subscribe/singlemeter.py diff --git a/tests/pytest/tools/taosdemoTestInterlace.py b/tests/pytest/tools/taosdemoTestInterlace.py index 9ceb8c5cbb..953bfff90f 100644 --- a/tests/pytest/tools/taosdemoTestInterlace.py +++ b/tests/pytest/tools/taosdemoTestInterlace.py @@ -17,6 +17,7 @@ from util.log import * from util.cases import * from util.sql import * from util.dnodes import * +import subprocess class TDTestCase: @@ -39,7 +40,7 @@ class TDTestCase: if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + buildPath = root[:len(root) - len("/build/bin")] break return buildPath @@ -50,14 +51,23 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - os.system("%staosdemo -f tools/insert-interlace.json" % binPath) + binPath = buildPath + "/build/bin/" + taosdemoCmd = "%staosdemo -f tools/insert-interlace.json -pp 2>&1 | grep sleep | wc -l" % binPath + sleepTimes = subprocess.check_output( + taosdemoCmd, shell=True).decode("utf-8") + print("sleep times: %d" % int(sleepTimes)) + + if (int(sleepTimes) != 16): + caller = inspect.getframeinfo(inspect.stack()[0][0]) + tdLog.exit( + "%s(%d) failed: expected sleep times 16, actual %d" % + (caller.filename, caller.lineno, int(sleepTimes))) tdSql.execute("use db") tdSql.query("select count(tbname) from db.stb") - tdSql.checkData(0, 0, 100) + tdSql.checkData(0, 0, 9) tdSql.query("select count(*) from db.stb") - tdSql.checkData(0, 0, 33000) + tdSql.checkData(0, 0, 2250) def stop(self): tdSql.close() -- GitLab