Skip to content

T
TDengine

慢慢CG / TDengine
与 Fork 源项目一致

Fork自 taosdata / TDengine

通知 1
Star 0
Fork 0
T
TDengine
提交 046ec16a 编写于 作者: haoranc's avatar haoranc
浏览文件
操作

[TD-4552]<test>: add testcase that compact wal file with replica

上级 2ff6b516
隐藏空白更改
内联 并排
Showing with 419 addition and 13 deletion
tests/pytest/wal/insertDataDb1Replica2.json 0 → 100644
浏览文件 @ 046ec16a
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 10,
"num_of_records_per_req": 1000,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db1",
"drop": "yes",
"replica": 2,
"days": 10,
"cache": 50,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb0",
"child_table_exists":"no",
"childtable_count": 1000,
"childtable_prefix": "stb00_",
"auto_create_table": "no",
"batch_create_tbl_num": 100,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 100,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
},
{
"name": "stb1",
"child_table_exists":"no",
"childtable_count": 1000,
"childtable_prefix": "stb01_",
"auto_create_table": "no",
"batch_create_tbl_num": 10,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 200,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
}]
}]
}
tests/pytest/wal/insertDataDb2NewstabReplica2.json 0 → 100644
浏览文件 @ 046ec16a
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 0,
"num_of_records_per_req": 3000,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db2",
"drop": "no",
"replica": 2,
"days": 10,
"cache": 50,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb0",
"child_table_exists":"no",
"childtable_count": 1,
"childtable_prefix": "stb0_",
"auto_create_table": "no",
"batch_create_tbl_num": 100,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 0,
"childtable_limit": -1,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
},
{
"name": "stb1",
"child_table_exists":"yes",
"childtable_count": 1,
"childtable_prefix": "stb01_",
"auto_create_table": "no",
"batch_create_tbl_num": 10,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 10,
"childtable_limit": -1,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-11-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
}]
}]
}
tests/pytest/wal/insertDataDb2Replica2.json 0 → 100644
浏览文件 @ 046ec16a
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 0,
"num_of_records_per_req": 3000,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db2",
"drop": "yes",
"replica": 2,
"days": 10,
"cache": 50,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb0",
"child_table_exists":"no",
"childtable_count": 2000,
"childtable_prefix": "stb0_",
"auto_create_table": "no",
"batch_create_tbl_num": 100,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 2000,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
},
{
"name": "stb1",
"child_table_exists":"no",
"childtable_count": 2,
"childtable_prefix": "stb1_",
"auto_create_table": "no",
"batch_create_tbl_num": 10,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 5,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
}]
}]
}
tests/pytest/wal/sdbComp.py
浏览文件 @ 046ec16a
...@@ -108,10 +108,10 @@ class TDTestCase: ...@@ -108,10 +108,10 @@ class TDTestCase:
tdSql.query("select count(*) from stb2_0") tdSql.query("select count(*) from stb2_0")
tdSql.checkData(0, 0, 2) tdSql.checkData(0, 0, 2)
# delete useless file
testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf ./insert_res.txt") os.system("rm -rf ./insert_res.txt")
os.system("rm -rf wal/sdbComp.py.sql") os.system("rm -rf wal/%s.sql" % testcaseFilename )
......
tests/pytest/wal/sdbCompCluster.py
浏览文件 @ 046ec16a
...@@ -61,7 +61,6 @@ class TwoClients: ...@@ -61,7 +61,6 @@ class TwoClients:
print(conn1) print(conn1)
cur1 = conn1.cursor() cur1 = conn1.cursor()
tdSql.init(cur1, True) tdSql.init(cur1, True)
# tdSql.init(cur2, True)
# new db and insert data # new db and insert data
os.system("rm -rf /var/lib/taos/mnode_bak/") os.system("rm -rf /var/lib/taos/mnode_bak/")
...@@ -87,17 +86,12 @@ class TwoClients: ...@@ -87,17 +86,12 @@ class TwoClients:
tdSql.execute("alter table stb2_0 add column col2 binary(4)") tdSql.execute("alter table stb2_0 add column col2 binary(4)")
tdSql.execute("alter table stb2_0 drop column col1") tdSql.execute("alter table stb2_0 drop column col1")
tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
print(conn2)
cur2 = conn2.cursor()
tdSql.init(cur2, True)
# stop taosd and compact wal file # stop taosd and compact wal file
os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -9") os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2")
sleep(2) sleep(10)
os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ") os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ")
sleep(5) sleep(10)
os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
sleep(4) sleep(4)
tdSql.execute("reset query cache") tdSql.execute("reset query cache")
...@@ -105,7 +99,17 @@ class TwoClients: ...@@ -105,7 +99,17 @@ class TwoClients:
print(query_pid2) print(query_pid2)
assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath
# new taos connecting to server
conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
print(conn2)
cur2 = conn2.cursor()
tdSql.init(cur2, True)
# use new wal file to start up tasod # use new wal file to start up tasod
tdSql.query("show databases")
for i in range(tdSql.queryRows):
if tdSql.queryResult[i][0]=="db2":
assert tdSql.queryResult[i][4]==1 , "replica is wrong"
tdSql.execute("use db2") tdSql.execute("use db2")
tdSql.query("select count (tbname) from stb0") tdSql.query("select count (tbname) from stb0")
tdSql.checkData(0, 0, 1) tdSql.checkData(0, 0, 1)
...@@ -117,7 +121,14 @@ class TwoClients: ...@@ -117,7 +121,14 @@ class TwoClients:
tdSql.checkData(0, 0, 2) tdSql.checkData(0, 0, 2)
tdSql.query("select count(*) from stb2_0") tdSql.query("select count(*) from stb2_0")
tdSql.checkData(0, 0, 2) tdSql.checkData(0, 0, 2)
tdSql.query("select * from stb2_0")
tdSql.checkData(1, 2, 'R')
# delete useless file
testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf ./insert_res.txt")
os.system("rm -rf wal/%s.sql" % testcaseFilename )
clients = TwoClients() clients = TwoClients()
clients.initConnection() clients.initConnection()
# clients.getBuildPath() # clients.getBuildPath()
......
tests/pytest/wal/sdbCompClusterReplica2.py 0 → 100644
浏览文件 @ 046ec16a
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
import sys
sys.path.insert(0, os.getcwd())
from util.log import *
from util.sql import *
from util.dnodes import *
import taos
import threading
class TwoClients:
def initConnection(self):
self.host = "chenhaoran02"
self.user = "root"
self.password = "taosdata"
self.config = "/etc/taos/"
self.port =6030
self.rowNum = 10
self.ts = 1537146000000
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
return buildPath
def run(self):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
binPath = buildPath+ "/build/bin/"
walFilePath = "/var/lib/taos/mnode_bak/wal/"
# new taos client
conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
print(conn1)
cur1 = conn1.cursor()
tdSql.init(cur1, True)
# new db and insert data
os.system("rm -rf /var/lib/taos/mnode_bak/")
os.system("rm -rf /var/lib/taos/mnode_temp/")
tdSql.execute("drop database if exists db2")
os.system("%staosdemo -f wal/insertDataDb1Replica2.json -y " % binPath)
tdSql.execute("drop database if exists db1")
os.system("%staosdemo -f wal/insertDataDb2Replica2.json -y " % binPath)
tdSql.execute("drop table if exists db2.stb0")
os.system("%staosdemo -f wal/insertDataDb2NewstabReplica2.json -y " % binPath)
query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
print(query_pid1)
tdSql.execute("use db2")
tdSql.execute("drop table if exists stb1_0")
tdSql.execute("drop table if exists stb1_1")
tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')")
tdSql.execute("alter table db2.stb0 add column col4 int")
tdSql.execute("alter table db2.stb0 drop column col2")
tdSql.execute("alter table db2.stb0 add tag t3 int")
tdSql.execute("alter table db2.stb0 drop tag t1")
tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ")
tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)")
tdSql.execute("alter table stb2_0 add column col2 binary(4)")
tdSql.execute("alter table stb2_0 drop column col1")
tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
# stop taosd and compact wal file
os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2")
sleep(10)
os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ")
sleep(10)
os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
sleep(4)
tdSql.execute("reset query cache")
query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
print(query_pid2)
assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath
# new taos connecting to server
conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
print(conn2)
cur2 = conn2.cursor()
tdSql.init(cur2, True)
# use new wal file to start up tasod
tdSql.query("show databases")
for i in range(tdSql.queryRows):
if tdSql.queryResult[i][0]=="db2":
assert tdSql.queryResult[i][4]==2 , "replica is wrong"
tdSql.execute("use db2")
tdSql.query("select count (tbname) from stb0")
tdSql.checkData(0, 0, 1)
tdSql.query("select count (tbname) from stb1")
tdSql.checkRows(0)
tdSql.query("select count(*) from stb0_0")
tdSql.checkData(0, 0, 2)
tdSql.query("select count(*) from stb0")
tdSql.checkData(0, 0, 2)
tdSql.query("select count(*) from stb2_0")
tdSql.checkData(0, 0, 2)
tdSql.query("select * from stb2_0")
tdSql.checkData(1, 2, 'R')
# delete useless file
testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf ./insert_res.txt")
os.system("rm -rf wal/%s.sql" % testcaseFilename )
clients = TwoClients()
clients.initConnection()
# clients.getBuildPath()
clients.run()
\ No newline at end of file
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或