Skip to content

T
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
T
TDengine
未验证 提交 f65674d1 编写于 作者: H Hui Li 提交者： GitHub
浏览文件
操作

Merge pull request #12756 from taosdata/test/chr/TD-14699 

test:add testcase of   multi-Thread inserting data
上级 3d0a2181 3c0a1bc7
隐藏空白更改
内联 并排
Showing with 53 addition and 69 deletion
tests/pytest/util/dnodes.py
浏览文件 @ f65674d1
...@@ -264,7 +264,7 @@ class TDDnode: ...@@ -264,7 +264,7 @@ class TDDnode:
cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
binPath, self.cfgDir) binPath, self.cfgDir)
else: else:
valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" valgrindCmdline = "valgrind --log-file=\"valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
cmd = "nohup %s %s -c %s 2>&1 & " % ( cmd = "nohup %s %s -c %s 2>&1 & " % (
valgrindCmdline, binPath, self.cfgDir) valgrindCmdline, binPath, self.cfgDir)
...@@ -325,7 +325,7 @@ class TDDnode: ...@@ -325,7 +325,7 @@ class TDDnode:
cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
binPath, self.cfgDir) binPath, self.cfgDir)
else: else:
valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" valgrindCmdline = "valgrind --log-file=\"valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
cmd = "nohup %s %s -c %s 2>&1 & " % ( cmd = "nohup %s %s -c %s 2>&1 & " % (
valgrindCmdline, binPath, self.cfgDir) valgrindCmdline, binPath, self.cfgDir)
......
tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim
浏览文件 @ f65674d1
...@@ -71,7 +71,7 @@ sql create database db replica $replica vgroups $vgroups ...@@ -71,7 +71,7 @@ sql create database db replica $replica vgroups $vgroups
$loop_cnt = 0 $loop_cnt = 0
check_db_ready: check_db_ready:
$loop_cnt = $loop_cnt + 1 $loop_cnt = $loop_cnt + 1
sleep 200 sleep 20
if $loop_cnt == 10 then if $loop_cnt == 10 then
print ====> db not ready! print ====> db not ready!
return -1 return -1
...@@ -83,7 +83,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat ...@@ -83,7 +83,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat
if $rows != 3 then if $rows != 3 then
return -1 return -1
endi endi
if $data(db)[19] != nostrict then if $data(db)[19] != ready then
goto check_db_ready goto check_db_ready
endi endi
...@@ -93,7 +93,7 @@ $loop_cnt = 0 ...@@ -93,7 +93,7 @@ $loop_cnt = 0
check_vg_ready: check_vg_ready:
$loop_cnt = $loop_cnt + 1 $loop_cnt = $loop_cnt + 1
sleep 200 sleep 200
if $loop_cnt == 40 then if $loop_cnt == 300 then
print ====> vgroups not ready! print ====> vgroups not ready!
return -1 return -1
endi endi
...@@ -175,6 +175,7 @@ while $i < $tbNum ...@@ -175,6 +175,7 @@ while $i < $tbNum
endw endw
$totalTblNum = $tbNum * 2 $totalTblNum = $tbNum * 2
print ====>totalTblNum:$totalTblNum
sql show tables sql show tables
if $rows != $totalTblNum then if $rows != $totalTblNum then
return -1 return -1
...@@ -226,7 +227,7 @@ $loop_cnt = 0 ...@@ -226,7 +227,7 @@ $loop_cnt = 0
check_vg_ready_2: check_vg_ready_2:
$loop_cnt = $loop_cnt + 1 $loop_cnt = $loop_cnt + 1
sleep 200 sleep 200
if $loop_cnt == 10 then if $loop_cnt == 300 then
print ====> vgroups switch fail!!! print ====> vgroups switch fail!!!
return -1 return -1
endi endi
......
tests/system-test/1-insert/insertWithMoreVgroup.py
浏览文件 @ f65674d1
...@@ -30,9 +30,12 @@ class TDTestCase: ...@@ -30,9 +30,12 @@ class TDTestCase:
# #
# --------------- main frame ------------------- # --------------- main frame -------------------
# #
clientCfgDict = {'queryproxy': '1'} clientCfgDict = {'queryproxy': '1','debugFlag': 135}
clientCfgDict["queryproxy"] = '2' clientCfgDict["queryproxy"] = '2'
clientCfgDict["debugFlag"] = 143
updatecfgDict = {'clientCfg': {}} updatecfgDict = {'clientCfg': {}}
updatecfgDict = {'debugFlag': 143}
updatecfgDict["clientCfg"] = clientCfgDict updatecfgDict["clientCfg"] = clientCfgDict
def caseDescription(self): def caseDescription(self):
''' '''
...@@ -116,7 +119,7 @@ class TDTestCase: ...@@ -116,7 +119,7 @@ class TDTestCase:
# tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) # tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate))
return return
def mutiThread_create_tables(self,host,dbname,stbname,vgroups,threadNumbers,count): def mutiThread_create_tables(self,host,dbname,stbname,vgroups,threadNumbers,childrowcount):
buildPath = self.getBuildPath() buildPath = self.getBuildPath()
config = buildPath+ "../sim/dnode1/cfg/" config = buildPath+ "../sim/dnode1/cfg/"
...@@ -125,7 +128,7 @@ class TDTestCase: ...@@ -125,7 +128,7 @@ class TDTestCase:
tsql.execute("drop database if exists %s"%dbname) tsql.execute("drop database if exists %s"%dbname)
tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) tsql.execute("create database %s vgroups %d"%(dbname,vgroups))
tsql.execute("use %s" %dbname) tsql.execute("use %s" %dbname)
count=int(count) count=int(childrowcount)
threads = [] threads = []
for i in range(threadNumbers): for i in range(threadNumbers):
tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i)) tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i))
...@@ -137,7 +140,7 @@ class TDTestCase: ...@@ -137,7 +140,7 @@ class TDTestCase:
tr.join() tr.join()
end_time = time.time() end_time = time.time()
spendTime=end_time-start_time spendTime=end_time-start_time
speedCreate=count/spendTime speedCreate=threadNumbers*count/spendTime
tdLog.debug("spent %.2fs to create %d stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,threadNumbers,threadNumbers*count,speedCreate)) tdLog.debug("spent %.2fs to create %d stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,threadNumbers,threadNumbers*count,speedCreate))
return return
...@@ -146,41 +149,39 @@ class TDTestCase: ...@@ -146,41 +149,39 @@ class TDTestCase:
# insert data # insert data
def insert_data(self, host, dbname, stbname, ts_start,rowCount): def insert_data(self, host, dbname, stbname, chilCount, ts_start, rowCount):
buildPath = self.getBuildPath() buildPath = self.getBuildPath()
config = buildPath+ "../sim/dnode1/cfg/" config = buildPath+ "../sim/dnode1/cfg/"
tsql=self.newcur(host,config) tsql=self.newcur(host,config)
tdLog.debug("ready to inser data") tdLog.debug("ready to inser data")
tsql.execute("use %s" %dbname) tsql.execute("use %s" %dbname)
pre_insert = "insert into " pre_insert = "insert into "
sql = pre_insert sql = pre_insert
tcount=int(tcount) chilCount=int(chilCount)
allRows=tcount*rowCount allRows=chilCount*rowCount
tdLog.debug("doing insert data into stable-index:%s rows:%d ..."%(stbname, allRows)) tdLog.debug("doing insert data into stable-index:%s rows:%d ..."%(stbname, allRows))
exeStartTime=time.time() exeStartTime=time.time()
for i in range(0,tcount): for i in range(0,chilCount):
sql += " %s_%d values "%(stbname,i) sql += " %s_%d values "%(stbname,i)
for j in range(rowCount): for j in range(rowCount):
sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j) sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j)
if j >0 and j%5000 == 0: if j >0 and j%4000 == 0:
# print(sql) # print(sql)
tdSql.execute(sql) tsql.execute(sql)
sql = "insert into %s_%d values " %(stbname,i) sql = "insert into %s_%d values " %(stbname,i)
# end sql # end sql
if sql != pre_insert: if sql != pre_insert:
# print(sql) # print(sql)
tdSql.execute(sql) print(len(sql))
tsql.execute(sql)
exeEndTime=time.time() exeEndTime=time.time()
spendTime=exeEndTime-exeStartTime spendTime=exeEndTime-exeStartTime
speedInsert=allRows/spendTime speedInsert=allRows/spendTime
# tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert)) tdLog.debug("spent %.2fs to INSERT %d rows into %s , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,stbname,speedInsert))
# tdLog.debug("INSERT TABLE DATA ............ [OK]")
tdLog.debug("INSERT TABLE DATA ............ [OK]")
return return
def mutiThread_insert_data(self, host, dbname, stbname, threadNumbers, ts_start, tcountStart,tcountStop,rowCount): def mutiThread_insert_data(self, host, dbname, stbname, threadNumbers, chilCount, ts_start, childrowcount):
buildPath = self.getBuildPath() buildPath = self.getBuildPath()
config = buildPath+ "../sim/dnode1/cfg/" config = buildPath+ "../sim/dnode1/cfg/"
...@@ -188,42 +189,11 @@ class TDTestCase: ...@@ -188,42 +189,11 @@ class TDTestCase:
tdLog.debug("ready to inser data") tdLog.debug("ready to inser data")
tsql.execute("use %s" %dbname) tsql.execute("use %s" %dbname)
pre_insert = "insert into " chilCount=int(chilCount)
sql = pre_insert
tcount=tcountStop-tcountStart
allRows=tcount*rowCount
tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbname, allRows))
exeStartTime=time.time()
for i in range(tcountStart,tcountStop):
sql += " %s_%d values "%(stbname,i)
for j in range(rowCount):
sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j)
if j >0 and j%5000 == 0:
# print(sql)
tdSql.execute(sql)
sql = "insert into %s_%d values " %(stbname,i)
# end sql
if sql != pre_insert:
# print(sql)
tdSql.execute(sql)
exeEndTime=time.time()
spendTime=exeEndTime-exeStartTime
speedInsert=allRows/spendTime
# tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert))
tdLog.debug("INSERT TABLE DATA ............ [OK]")
buildPath = self.getBuildPath()
config = buildPath+ "../sim/dnode1/cfg/"
tsql=self.newcur(host,config)
tsql.execute("use %s" %dbname)
count=int(count)
threads = [] threads = []
for i in range(threadNumbers): for i in range(threadNumbers):
tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i)) # tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i))
threads.append(thd.Thread(target=self.create_tables, args=(host, dbname, stbname+"%d"%i, count,))) threads.append(thd.Thread(target=self.insert_data, args=(host, dbname, stbname+"%d"%i, chilCount, ts_start, childrowcount,)))
start_time = time.time() start_time = time.time()
for tr in threads: for tr in threads:
tr.start() tr.start()
...@@ -231,8 +201,18 @@ class TDTestCase: ...@@ -231,8 +201,18 @@ class TDTestCase:
tr.join() tr.join()
end_time = time.time() end_time = time.time()
spendTime=end_time-start_time spendTime=end_time-start_time
speedCreate=count/spendTime tableCounts=threadNumbers*chilCount
tdLog.debug("spent %.2fs to create %d stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,threadNumbers,threadNumbers*count,speedCreate)) stableRows=chilCount*childrowcount
allRows=stableRows*threadNumbers
speedInsert=allRows/spendTime
for i in range(threadNumbers):
tdSql.execute("use %s" %dbname)
tdSql.query("select count(*) from %s%d"%(stbname,i))
tdSql.checkData(0,0,stableRows)
tdLog.debug("spent %.2fs to insert %d rows into %d stable and %d table, speed is %.2f table/s... [OK]"% (spendTime,allRows,threadNumbers,tableCounts,speedInsert))
tdLog.debug("INSERT TABLE DATA ............ [OK]")
return return
...@@ -288,7 +268,10 @@ class TDTestCase: ...@@ -288,7 +268,10 @@ class TDTestCase:
def test_case1(self): def test_case1(self):
tdLog.debug("-----create database and muti-thread create tables test------- ") tdLog.debug("-----create database and muti-thread create tables test------- ")
#host,dbname,stbname,vgroups,threadNumbers,tcountStart,tcountStop #host,dbname,stbname,vgroups,threadNumbers,tcountStart,tcountStop
self.mutiThread_create_tables(host="localhost",dbname="db2",stbname="stb2", vgroups=1, threadNumbers=5, count=10000) #host, dbname, stbname, threadNumbers, chilCount, ts_start, childrowcount
self.mutiThread_create_tables(host="localhost",dbname="db",stbname="stb", vgroups=1, threadNumbers=5, childrowcount=50)
self.mutiThread_insert_data(host="localhost",dbname="db",stbname="stb", threadNumbers=5,chilCount=50,ts_start=self.ts,childrowcount=10)
return return
# test case2 base:insert data # test case2 base:insert data
...@@ -366,17 +349,17 @@ class TDTestCase: ...@@ -366,17 +349,17 @@ class TDTestCase:
# run case # run case
def run(self): def run(self):
# # test base case # create database and tables。
# self.test_case1() self.test_case1()
# tdLog.debug(" LIMIT test_case1 ............ [OK]") tdLog.debug(" LIMIT test_case1 ............ [OK]")
# test case # # taosBenchmark ： create database and table
# self.test_case2() # self.test_case2()
# tdLog.debug(" LIMIT test_case2 ............ [OK]") # tdLog.debug(" LIMIT test_case2 ............ [OK]")
# test case # # taosBenchmark：create database/table and insert data
self.test_case3() # self.test_case3()
tdLog.debug(" LIMIT test_case3 ............ [OK]") # tdLog.debug(" LIMIT test_case3 ............ [OK]")
# # test qnode # # test qnode
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或