diff --git a/tests/perftest-scripts/taosadapter_perftest/opentsdb_json_jmeter_csv_import.jmx b/tests/perftest-scripts/taosadapter_perftest/opentsdb_json_jmeter_csv_import.jmx
new file mode 100644
index 0000000000000000000000000000000000000000..7192421de4ca8f88f86226852bbe7242423efe65
--- /dev/null
+++ b/tests/perftest-scripts/taosadapter_perftest/opentsdb_json_jmeter_csv_import.jmx
@@ -0,0 +1,203 @@
+
+
+
+
+
+ false
+ true
+ false
+
+
+
+
+
+
+
+ continue
+
+ false
+ 1
+
+ 1
+ 1
+ false
+
+
+ true
+
+
+
+ true
+
+
+
+ false
+ create database if not exists test precision 'ms'
+ =
+
+
+
+
+
+
+
+ http://192.168.1.85:6041/rest/sql
+ POST
+ true
+ false
+ true
+ false
+
+
+
+
+
+
+ false
+
+ saveConfig
+
+
+ true
+ true
+ true
+
+ true
+ true
+ true
+ true
+ false
+ true
+ true
+ false
+ false
+ false
+ true
+ false
+ false
+ false
+ true
+ 0
+ true
+ true
+ true
+ true
+ true
+ true
+
+
+
+
+
+
+
+
+ Authorization
+ Basic cm9vdDp0YW9zZGF0YQ==
+
+
+
+
+
+
+ continue
+
+ false
+ looptimes
+
+ 100
+
+ false
+
+
+ true
+
+
+
+ true
+
+
+
+ false
+ {"metric": "cpu.usage_user.rows", "timestamp":${ts_csv_count}, "value":32.261068286779754, "tags":{"arch":"x64","datacenter":"us-west-1b","hostname":"host_5","os":"Ubuntu16","rack":"13","region":"us-west-1","service":"10","service_environment":"staging","service_version":"${row_csv_count}","team":"NYC"}}
+ =
+
+
+
+
+
+
+
+ http://192.168.1.85:6041/opentsdb/v1/put/json/test
+ POST
+ true
+ false
+ true
+ false
+
+
+
+
+
+
+ false
+
+ saveConfig
+
+
+ true
+ true
+ true
+
+ true
+ true
+ true
+ true
+ false
+ true
+ true
+ false
+ false
+ false
+ true
+ false
+ false
+ false
+ true
+ 0
+ true
+ true
+ true
+ true
+ true
+ true
+
+
+
+
+
+
+
+
+ Authorization
+ Basic cm9vdDp0YW9zZGF0YQ==
+
+
+
+
+
+
+ ,
+ UTF-8
+ import_file_name
+ false
+ false
+ true
+ shareMode.all
+ false
+ ts_csv_count,row_csv_count
+
+
+
+
+
diff --git a/tests/perftest-scripts/taosadapter_perftest/opentsdb_telnet_jmeter_csv_import.jmx b/tests/perftest-scripts/taosadapter_perftest/opentsdb_telnet_jmeter_csv_import.jmx
new file mode 100644
index 0000000000000000000000000000000000000000..672c377fcad2e988443efa28ae10c0b3c09152be
--- /dev/null
+++ b/tests/perftest-scripts/taosadapter_perftest/opentsdb_telnet_jmeter_csv_import.jmx
@@ -0,0 +1,203 @@
+
+
+
+
+
+ false
+ true
+ false
+
+
+
+
+
+
+
+ continue
+
+ false
+ 1
+
+ 1
+ 1
+ false
+
+
+ true
+
+
+
+ true
+
+
+
+ false
+ create database if not exists test precision 'ms'
+ =
+
+
+
+
+
+
+
+ http://192.168.1.85:6041/rest/sql
+ POST
+ true
+ false
+ true
+ false
+
+
+
+
+
+
+ false
+
+ saveConfig
+
+
+ true
+ true
+ true
+
+ true
+ true
+ true
+ true
+ false
+ true
+ true
+ false
+ false
+ false
+ true
+ false
+ false
+ false
+ true
+ 0
+ true
+ true
+ true
+ true
+ true
+ true
+
+
+
+
+
+
+
+
+ Authorization
+ Basic cm9vdDp0YW9zZGF0YQ==
+
+
+
+
+
+
+ continue
+
+ false
+ looptimes
+
+ 100
+
+ false
+
+
+ true
+
+
+
+ true
+
+
+
+ false
+ cpu.usage_user.rows ${ts_csv_count} 22.345567 arch=x64 datacenter=us-west-1b hostname=host_5 os=Ubuntu16 rack=13 region=us-west-1 service=10 service_environment=staging service_version=${row_csv_count} team=NYC
+ =
+
+
+
+
+
+
+
+ http://192.168.1.85:6041/opentsdb/v1/put/telnet/test
+ POST
+ true
+ false
+ true
+ false
+
+
+
+
+
+
+ false
+
+ saveConfig
+
+
+ true
+ true
+ true
+
+ true
+ true
+ true
+ true
+ false
+ true
+ true
+ false
+ false
+ false
+ true
+ false
+ false
+ false
+ true
+ 0
+ true
+ true
+ true
+ true
+ true
+ true
+
+
+
+
+
+
+
+
+ Authorization
+ Basic cm9vdDp0YW9zZGF0YQ==
+
+
+
+
+
+
+ ,
+ UTF-8
+ import_file_name
+ false
+ false
+ true
+ shareMode.all
+ false
+ ts_csv_count,row_csv_count
+
+
+
+
+
diff --git a/tests/perftest-scripts/taosadapter_perftest/taosadapter_perftest.py b/tests/perftest-scripts/taosadapter_perftest/taosadapter_perftest.py
index 06947d47925336032c5250e6ee9b684627359863..bf621b6bb55ecb5b9db3cc6c42937c785cd62960 100644
--- a/tests/perftest-scripts/taosadapter_perftest/taosadapter_perftest.py
+++ b/tests/perftest-scripts/taosadapter_perftest/taosadapter_perftest.py
@@ -13,12 +13,14 @@ class TaosadapterPerftest():
self.telnetCreateStbJmxFile = "opentsdb_telnet_createStb.jmx"
self.telnetCreateTbJmxFile = "opentsdb_telnet_createTb.jmx"
self.telnetInsertRowsFile = "opentsdb_telnet_insertRows.jmx"
- self.telnetMixJmxFile = "opentsdb_telnet_MixTbRows.jmx"
+ # self.telnetMixJmxFile = "opentsdb_telnet_MixTbRows.jmx"
+ self.telnetMixJmxFile = "opentsdb_telnet_jmeter_csv_import.jmx"
self.jsonCreateStbJmxFile = "opentsdb_json_createStb.jmx"
self.jsonCreateTbJmxFile = "opentsdb_json_createTb.jmx"
self.jsonInsertRowsFile = "opentsdb_json_insertRows.jmx"
- self.jsonMixJmxFile = "opentsdb_json_MixTbRows.jmx"
+ # self.jsonMixJmxFile = "opentsdb_json_MixTbRows.jmx"
+ self.jsonMixJmxFile = "opentsdb_json_jmeter_csv_import.jmx"
self.logfile = "taosadapter_perftest.log"
self.createStbThreads = 100
@@ -46,7 +48,7 @@ class TaosadapterPerftest():
result = os.popen(shell_cmd).read().strip()
return result
- def modifyJxmLooptimes(self, filename, looptimes, row_count=None):
+ def modifyJxmLooptimes(self, filename, looptimes, row_count=None, import_file_name=None):
'''
modify looptimes
'''
@@ -59,6 +61,9 @@ class TaosadapterPerftest():
if row_count is not None:
if "row_count" in line:
line = line.replace("row_count", row_count)
+ if import_file_name is not None:
+ if "import_file_name" in line:
+ line = line.replace("import_file_name", import_file_name)
f_w.write(line)
def cleanAndRestartTaosd(self):
@@ -89,6 +94,16 @@ class TaosadapterPerftest():
f.seek(0)
f.truncate()
+ def genMixTbRows(self, filename, table_count, row_count):
+ logger.info('generating import data file')
+ ts_start = 1614530008000
+ with open(filename, "w", encoding="utf-8") as f_w:
+ for i in range(table_count):
+ for j in range(row_count):
+ input_line = str(ts_start) + "," + str(i) + '\n'
+ ts_start += 1
+ f_w.write(input_line)
+
def outputParams(self, protocol, create_type):
'''
procotol is "telnet" or "json"
@@ -125,26 +140,47 @@ class TaosadapterPerftest():
shutil.copyfile(jmxfile, handle_file)
replace_count = int(count/threads)
self.modifyJxmLooptimes(handle_file, str(replace_count))
+ logger.info(f'jmeter running ----- jmeter -n -t {handle_file} -l {report_dir}/{handle_file}.txt -e -o {report_dir}')
result = self.exec_local_cmd(f"jmeter -n -t {handle_file} -l {report_dir}/{handle_file}.txt -e -o {report_dir}")
logger.info(result)
logger.info("----- sleep 120s and please record data -----")
time.sleep(120)
- def insertMixTbRows(self, procotol, looptimes, row_count):
+ def insertMixTbRows(self, procotol, table_count, row_count):
self.cleanAndRestartTaosd()
- jmxfile = f"opentsdb_{procotol}_{looptimes}Tb100Rows.jmx"
+ local_path = os.getcwd()
+ jmxfile = f"opentsdb_{procotol}_{table_count}Tb{row_count}Rows.jmx"
+ import_file_name = f"import_opentsdb_{procotol}_{table_count}Tb{row_count}Rows.txt"
+ import_file_path = local_path + '/' + import_file_name
+ self.genMixTbRows(import_file_name, table_count, row_count)
report_dir = f'testreport/{jmxfile}'
self.recreateReportDir(report_dir)
if procotol == "telnet":
shutil.copyfile(self.telnetMixJmxFile, jmxfile)
else:
shutil.copyfile(self.jsonMixJmxFile, jmxfile)
-
- self.modifyJxmLooptimes(jmxfile, str(looptimes), str(row_count))
+ self.modifyJxmLooptimes(jmxfile, str(int(table_count*row_count/100)), import_file_name=import_file_path)
+ logger.info(f'jmeter running ----- jmeter -n -t {jmxfile} -l {report_dir}/{jmxfile}.txt -e -o {report_dir}')
result = self.exec_local_cmd(f"jmeter -n -t {jmxfile} -l {report_dir}/{jmxfile}.txt -e -o {report_dir}")
logger.info(result)
logger.info("----- sleep 120s and please record data -----")
time.sleep(120)
+
+ # def insertMixTbRows(self, procotol, looptimes, row_count):
+ # self.cleanAndRestartTaosd()
+ # jmxfile = f"opentsdb_{procotol}_{looptimes}Tb100Rows.jmx"
+ # report_dir = f'testreport/{jmxfile}'
+ # self.recreateReportDir(report_dir)
+ # if procotol == "telnet":
+ # shutil.copyfile(self.telnetMixJmxFile, jmxfile)
+ # else:
+ # shutil.copyfile(self.jsonMixJmxFile, jmxfile)
+
+ # self.modifyJxmLooptimes(jmxfile, str(looptimes), str(row_count))
+ # result = self.exec_local_cmd(f"jmeter -n -t {jmxfile} -l {report_dir}/{jmxfile}.txt -e -o {report_dir}")
+ # logger.info(result)
+ # logger.info("----- sleep 120s and please record data -----")
+ # time.sleep(120)
@@ -154,51 +190,35 @@ if __name__ == '__main__':
logger.info('------------ Start testing the scenarios in the report chapter 3.4.1 ------------')
for procotol in ["telnet", "json"]:
- # logger.info(f'----- {procotol} protocol ------- Creating 30W stable ------------')
- # taosadapterPerftest.insertTDengine(procotol, "stb", 300000)
- # logger.info(f'----- {procotol} protocol ------- Creating 100W table with stb "cpu.usage_user" ------------')
- # taosadapterPerftest.insertTDengine(procotol, "tb", 1000000)
- # logger.info(f'----- {procotol} protocol ------- inserting 100W rows ------------')
- # taosadapterPerftest.insertTDengine(procotol, "rows", 1000000)
-
- # logger.info(f'----- {procotol} protocol ------- Creating 50W stable ------------')
- # taosadapterPerftest.insertTDengine(procotol, "stb", 500000)
- # logger.info(f'----- {procotol} protocol ------- Creating 500W table with stb "cpu.usage_user" ------------')
- # taosadapterPerftest.insertTDengine(procotol, "tb", 5000000)
- # logger.info(f'----- {procotol} protocol ------- inserting 500W rows ------------')
- # taosadapterPerftest.insertTDengine(procotol, "rows", 5000000)
-
- # logger.info(f'----- {procotol} protocol ------- Creating 100W stable ------------')
- # taosadapterPerftest.insertTDengine(procotol, "stb", 1000000)
- # logger.info(f'----- {procotol} protocol ------- Creating 1000W table with stb "cpu.usage_user" ------------')
- # taosadapterPerftest.insertTDengine(procotol, "tb", 10000000)
- # logger.info(f'----- {procotol} protocol ------- inserting 1000W rows ------------')
- # taosadapterPerftest.insertTDengine(procotol, "rows", 10000000)
+ logger.info(f'----- {procotol} protocol ------- Creating 30W stable ------------')
+ taosadapterPerftest.insertTDengine(procotol, "stb", 300000)
+ logger.info(f'----- {procotol} protocol ------- Creating 100W table with stb "cpu.usage_user" ------------')
+ taosadapterPerftest.insertTDengine(procotol, "tb", 1000000)
+ logger.info(f'----- {procotol} protocol ------- inserting 100W rows ------------')
+ taosadapterPerftest.insertTDengine(procotol, "rows", 1000000)
+
+ logger.info(f'----- {procotol} protocol ------- Creating 50W stable ------------')
+ taosadapterPerftest.insertTDengine(procotol, "stb", 500000)
+ logger.info(f'----- {procotol} protocol ------- Creating 500W table with stb "cpu.usage_user" ------------')
+ taosadapterPerftest.insertTDengine(procotol, "tb", 5000000)
+ logger.info(f'----- {procotol} protocol ------- inserting 500W rows ------------')
+ taosadapterPerftest.insertTDengine(procotol, "rows", 5000000)
+
+ logger.info(f'----- {procotol} protocol ------- Creating 100W stable ------------')
+ taosadapterPerftest.insertTDengine(procotol, "stb", 1000000)
+ logger.info(f'----- {procotol} protocol ------- Creating 1000W table with stb "cpu.usage_user" ------------')
+ taosadapterPerftest.insertTDengine(procotol, "tb", 10000000)
+ logger.info(f'----- {procotol} protocol ------- inserting 1000W rows ------------')
+ taosadapterPerftest.insertTDengine(procotol, "rows", 10000000)
+
+ logger.info(f'----- {procotol} protocol ------- Creating 10W stable 1000Rows ------------')
+ taosadapterPerftest.insertMixTbRows(procotol, 100000, 1000)
logger.info(f'----- {procotol} protocol ------- Creating 100W stable 100Rows ------------')
- taosadapterPerftest.insertMixTbRows(procotol, 1000000, 1000000)
-
- logger.info(f'----- {procotol} protocol ------- Creating 1000W stable 100Rows ------------')
- taosadapterPerftest.insertMixTbRows(procotol, 10000000, 10000000)
+ taosadapterPerftest.insertMixTbRows(procotol, 1000000, 100)
- logger.info(f'----- {procotol} protocol ------- Creating 3000W stable 100Rows ------------')
- taosadapterPerftest.insertMixTbRows(procotol, 30000000, 30000000)
+ logger.info(f'----- {procotol} protocol ------- Creating 000W stable 100Rows ------------')
+ taosadapterPerftest.insertMixTbRows(procotol, 5000000, 20)
logger.info(f'----- {procotol} protocol ------- Creating 5000W stable 100Rows ------------')
- taosadapterPerftest.insertMixTbRows(procotol, 50000000, 50000000)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+ taosadapterPerftest.insertMixTbRows(procotol, 10000000, 10)