Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
冷面1992
TDengine
提交
8f86e3bd
TDengine
项目概览
冷面1992
/
TDengine
与 Fork 源项目一致
Fork自
taosdata / TDengine
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
8f86e3bd
编写于
3月 31, 2021
作者:
H
huili
提交者:
GitHub
3月 31, 2021
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #5631 from taosdata/xiaoping/add_test_case
[TD-3602]<test>: update perf test script
上级
3fca391b
3631d276
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
114 addition
and
84 deletion
+114
-84
tests/perftest-scripts/perftest-query.sh
tests/perftest-scripts/perftest-query.sh
+3
-16
tests/pytest/tools/taosdemoPerformance.py
tests/pytest/tools/taosdemoPerformance.py
+105
-62
tests/test-all.sh
tests/test-all.sh
+6
-6
未找到文件。
tests/perftest-scripts/perftest-query.sh
浏览文件 @
8f86e3bd
...
...
@@ -50,7 +50,7 @@ function buildTDengine {
echo
"repo up-to-date"
else
echo
"repo need to pull"
git pull
>
/dev/null
git pull
>
/dev/null
2>&1
LOCAL_COMMIT
=
`
git rev-parse
--short
@
`
cd
debug
...
...
@@ -71,27 +71,14 @@ function runQueryPerfTest {
python3 query/queryPerformance.py
-c
$LOCAL_COMMIT
|
tee
-a
$PERFORMANCE_TEST_REPORT
python3 insert/insertFromCSVPerformance.py
-c
$LOCAL_COMMIT
|
tee
-a
$PERFORMANCE_TEST_REPORT
yes
| taosdemo
-c
/etc/taosperf/
-d
taosdemo_insert_test
-x
>
taosdemoperf.txt
CREATETABLETIME
=
`
grep
'Spent'
taosdemoperf.txt |
awk
'NR==1{print $2}'
`
INSERTRECORDSTIME
=
`
grep
'Spent'
taosdemoperf.txt |
awk
'NR==2{print $2}'
`
REQUESTSPERSECOND
=
`
grep
'Spent'
taosdemoperf.txt |
awk
'NR==2{print $13}'
`
delay
=
`
grep
'delay'
taosdemoperf.txt |
awk
'{print $4}'
`
AVGDELAY
=
`
echo
${
delay
:0:
${#
delay
}
-3
}
`
delay
=
`
grep
'delay'
taosdemoperf.txt |
awk
'{print $6}'
`
MAXDELAY
=
`
echo
${
delay
:0:
${#
delay
}
-3
}
`
delay
=
`
grep
'delay'
taosdemoperf.txt |
awk
'{print $8}'
`
MINDELAY
=
`
echo
${
delay
:0:
${#
delay
}
-2
}
`
python3 tools/taosdemoPerformance.py
-c
$LOCAL_COMMIT
-t
$CREATETABLETIME
-i
$INSERTRECORDSTIME
-r
$REQUESTSPERSECOND
-avg
$AVGDELAY
-max
$MAXDELAY
-min
$MINDELAY
|
tee
-a
$PERFORMANCE_TEST_REPORT
[
-f
taosdemoperf.txt
]
&&
rm
taosdemoperf.txt
python3 tools/taosdemoPerformance.py
-c
$LOCAL_COMMIT
|
tee
-a
$PERFORMANCE_TEST_REPORT
}
function
sendReport
{
echo
"send report"
receiver
=
"
pxiao
@taosdata.com"
receiver
=
"
develop
@taosdata.com"
mimebody
=
"MIME-Version: 1.0
\n
Content-Type: text/html; charset=utf-8
\n
"
cd
$TDENGINE_DIR
...
...
tests/pytest/tools/taosdemoPerformance.py
浏览文件 @
8f86e3bd
...
...
@@ -11,26 +11,16 @@
# -*- coding: utf-8 -*-
import
sys
import
taos
import
time
import
datetime
import
csv
import
random
import
pandas
as
pd
import
argparse
import
os.path
import
json
class
taosdemoPerformace
:
def
__init__
(
self
,
commitID
,
dbName
,
createTableTime
,
insertRecordsTime
,
recordsPerSecond
,
avgDelay
,
maxDelay
,
minDelay
):
def
__init__
(
self
,
commitID
,
dbName
):
self
.
commitID
=
commitID
self
.
dbName
=
dbName
self
.
createTableTime
=
createTableTime
self
.
insertRecordsTime
=
insertRecordsTime
self
.
recordsPerSecond
=
recordsPerSecond
self
.
avgDelay
=
avgDelay
self
.
maxDelay
=
maxDelay
self
.
minDelay
=
minDelay
self
.
dbName
=
dbName
self
.
host
=
"127.0.0.1"
self
.
user
=
"root"
self
.
password
=
"taosdata"
...
...
@@ -39,8 +29,97 @@ class taosdemoPerformace:
self
.
host
,
self
.
user
,
self
.
password
,
self
.
config
)
self
.
config
)
self
.
insertDB
=
"insertDB"
;
def
generateJson
(
self
):
db
=
{
"name"
:
"%s"
%
self
.
insertDB
,
"drop"
:
"yes"
,
"replica"
:
1
}
stb
=
{
"name"
:
"meters"
,
"child_table_exists"
:
"no"
,
"childtable_count"
:
10000
,
"childtable_prefix"
:
"stb_"
,
"auto_create_table"
:
"no"
,
"data_source"
:
"rand"
,
"batch_create_tbl_num"
:
10
,
"insert_mode"
:
"taosc"
,
"insert_rows"
:
100000
,
"multi_thread_write_one_tbl"
:
"no"
,
"number_of_tbl_in_one_sql"
:
0
,
"rows_per_tbl"
:
100
,
"max_sql_len"
:
1024000
,
"disorder_ratio"
:
0
,
"disorder_range"
:
1000
,
"timestamp_step"
:
1
,
"start_timestamp"
:
"2020-10-01 00:00:00.000"
,
"sample_format"
:
"csv"
,
"sample_file"
:
"./sample.csv"
,
"tags_file"
:
""
,
"columns"
:
[
{
"type"
:
"INT"
,
"count"
:
4
}
],
"tags"
:
[
{
"type"
:
"INT"
,
"count"
:
1
},
{
"type"
:
"BINARY"
,
"len"
:
16
}
]
}
stables
=
[]
stables
.
append
(
stb
)
db
=
{
"dbinfo"
:
db
,
"super_tables"
:
stables
}
insert_data
=
{
"filetype"
:
"insert"
,
"cfgdir"
:
"/etc/taosperf"
,
"host"
:
"127.0.0.1"
,
"port"
:
6030
,
"user"
:
"root"
,
"password"
:
"taosdata"
,
"thread_count"
:
10
,
"thread_count_create_tbl"
:
10
,
"result_file"
:
"./insert_res.txt"
,
"confirm_parameter_prompt"
:
"no"
,
"insert_interval"
:
0
,
"num_of_records_per_req"
:
30000
,
"databases"
:
[
db
]
}
insert_json_file
=
f
"/tmp/insert.json"
with
open
(
insert_json_file
,
'w'
)
as
f
:
json
.
dump
(
insert_data
,
f
)
return
insert_json_file
def
getCMDOutput
(
self
,
cmd
):
cmd
=
os
.
popen
(
cmd
)
output
=
cmd
.
read
()
cmd
.
close
()
return
output
def
insertData
(
self
):
os
.
system
(
"taosdemo -f %s > taosdemoperf.txt"
%
self
.
generateJson
())
self
.
createTableTime
=
self
.
getCMDOutput
(
"grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'"
)
self
.
insertRecordsTime
=
self
.
getCMDOutput
(
"grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'"
)
self
.
recordsPerSecond
=
self
.
getCMDOutput
(
"grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'"
)
self
.
commitID
=
self
.
getCMDOutput
(
"git rev-parse --short HEAD"
)
delay
=
self
.
getCMDOutput
(
"grep 'delay' taosdemoperf.txt | awk '{print $4}'"
)
self
.
avgDelay
=
delay
[:
-
4
]
delay
=
self
.
getCMDOutput
(
"grep 'delay' taosdemoperf.txt | awk '{print $6}'"
)
self
.
maxDelay
=
delay
[:
-
4
]
delay
=
self
.
getCMDOutput
(
"grep 'delay' taosdemoperf.txt | awk '{print $8}'"
)
self
.
minDelay
=
delay
[:
-
3
]
os
.
system
(
"[ -f taosdemoperf.txt ] && rm taosdemoperf.txt"
)
def
createTablesAndStoreData
(
self
):
cursor
=
self
.
conn
.
cursor
()
...
...
@@ -48,14 +127,15 @@ class taosdemoPerformace:
cursor
.
execute
(
"use %s"
%
self
.
dbName
)
cursor
.
execute
(
"create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)"
)
print
(
"==================== taosdemo performance ===================="
)
print
(
"create tables time: %f"
%
self
.
createTableTime
)
print
(
"insert records time: %f"
%
self
.
insertRecordsTime
)
print
(
"records per second: %f"
%
self
.
recordsPerSecond
)
print
(
"avg delay: %f"
%
self
.
avgDelay
)
print
(
"max delay: %f"
%
self
.
maxDelay
)
print
(
"min delay: %f"
%
self
.
minDelay
)
cursor
.
execute
(
"insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)"
%
(
self
.
createTableTime
,
self
.
insertRecordsTime
,
self
.
recordsPerSecond
,
self
.
commitID
,
self
.
avgDelay
,
self
.
maxDelay
,
self
.
minDelay
))
cursor
.
execute
(
"drop database if exists taosdemo_insert_test"
)
print
(
"create tables time: %f"
%
float
(
self
.
createTableTime
))
print
(
"insert records time: %f"
%
float
(
self
.
insertRecordsTime
))
print
(
"records per second: %f"
%
float
(
self
.
recordsPerSecond
))
print
(
"avg delay: %f"
%
float
(
self
.
avgDelay
))
print
(
"max delay: %f"
%
float
(
self
.
maxDelay
))
print
(
"min delay: %f"
%
float
(
self
.
minDelay
))
cursor
.
execute
(
"insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)"
%
(
float
(
self
.
createTableTime
),
float
(
self
.
insertRecordsTime
),
float
(
self
.
recordsPerSecond
),
self
.
commitID
,
float
(
self
.
avgDelay
),
float
(
self
.
maxDelay
),
float
(
self
.
minDelay
)))
cursor
.
execute
(
"drop database if exists %s"
%
self
.
insertDB
)
cursor
.
close
()
...
...
@@ -64,7 +144,7 @@ if __name__ == '__main__':
parser
.
add_argument
(
'-c'
,
'--commit-id'
,
action
=
'store'
,
action
=
'store'
,
type
=
str
,
help
=
'git commit id (default: null)'
)
parser
.
add_argument
(
...
...
@@ -74,46 +154,9 @@ if __name__ == '__main__':
default
=
'perf'
,
type
=
str
,
help
=
'Database name to be created (default: perf)'
)
parser
.
add_argument
(
'-t'
,
'--create-table'
,
action
=
'store'
,
type
=
float
,
help
=
'create table time'
)
parser
.
add_argument
(
'-i'
,
'--insert-records'
,
action
=
'store'
,
type
=
float
,
help
=
'insert records time'
)
parser
.
add_argument
(
'-r'
,
'---records-per-second'
,
action
=
'store'
,
type
=
float
,
help
=
'records per request'
)
parser
.
add_argument
(
'-avg'
,
'---avg-delay'
,
action
=
'store'
,
type
=
float
,
help
=
'avg delay'
)
parser
.
add_argument
(
'-max'
,
'---max-delay'
,
action
=
'store'
,
type
=
float
,
help
=
'max delay'
)
parser
.
add_argument
(
'-min'
,
'---min-delay'
,
action
=
'store'
,
type
=
float
,
help
=
'min delay'
)
args
=
parser
.
parse_args
()
perftest
=
taosdemoPerformace
(
args
.
commit_id
,
args
.
database_name
,
args
.
create_table
,
args
.
insert_records
,
args
.
records_per_second
,
args
.
avg_delay
,
args
.
max_delay
,
args
.
min_delay
)
perftest
=
taosdemoPerformace
(
args
.
commit_id
,
args
.
database_name
)
perftest
.
insertData
()
perftest
.
createTablesAndStoreData
()
\ No newline at end of file
tests/test-all.sh
浏览文件 @
8f86e3bd
...
...
@@ -463,10 +463,10 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" !=
echo
"Running tests"
./apitest
>
/dev/null 2>&1
if
[
$?
!=
"0"
]
;
then
echo
"
prepare
failed"
echo
"
apitest
failed"
totalExampleFailed
=
`
expr
$totalExampleFailed
+ 1
`
else
echo
"
prepare
pass"
echo
"
apitest
pass"
totalExamplePass
=
`
expr
$totalExamplePass
+ 1
`
fi
...
...
@@ -481,19 +481,19 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" !=
./subscribe
-test
>
/dev/null 2>&1
if
[
$?
!=
"0"
]
;
then
echo
"
prepar
e failed"
echo
"
subscrib
e failed"
totalExampleFailed
=
`
expr
$totalExampleFailed
+ 1
`
else
echo
"
prepar
e pass"
echo
"
subscrib
e pass"
totalExamplePass
=
`
expr
$totalExamplePass
+ 1
`
fi
yes
|
./asyncdemo 127.0.0.1
test
1000 10
>
/dev/null 2>&1
if
[
$?
!=
"0"
]
;
then
echo
"
prepare
failed"
echo
"
asyncdemo
failed"
totalExampleFailed
=
`
expr
$totalExampleFailed
+ 1
`
else
echo
"
prepare
pass"
echo
"
asyncdemo
pass"
totalExamplePass
=
`
expr
$totalExamplePass
+ 1
`
fi
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录