Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Xiaomi
soar
提交
08cf841e
S
soar
项目概览
Xiaomi
/
soar
大约 1 年 前同步成功
通知
393
Star
8512
Fork
1328
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
S
soar
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
08cf841e
编写于
10月 27, 2018
作者:
martianzhang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
cleanup-test-database add test case
上级
99d9b11d
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
56 addition
and
23 deletion
+56
-23
cmd/soar/soar.go
cmd/soar/soar.go
+6
-5
common/config.go
common/config.go
+4
-4
env/env.go
env/env.go
+20
-13
env/env_test.go
env/env_test.go
+26
-1
未找到文件。
cmd/soar/soar.go
浏览文件 @
08cf841e
...
@@ -80,16 +80,17 @@ func main() {
...
@@ -80,16 +80,17 @@ func main() {
// 环境初始化,连接检查线上环境+构建测试环境
// 环境初始化,连接检查线上环境+构建测试环境
vEnv
,
rEnv
:=
env
.
BuildEnv
()
vEnv
,
rEnv
:=
env
.
BuildEnv
()
//如果使用CleanTestDataBase命令,则清除前1小时的数据库
if
common
.
Config
.
CleanTestDataBase
{
vEnv
.
CleanTestDataBase
()
}
// 如果使用到测试环境,在这里环境清理
// 如果使用到测试环境,在这里环境清理
if
common
.
Config
.
DropTestTemporary
{
if
common
.
Config
.
DropTestTemporary
{
defer
vEnv
.
CleanUp
()
defer
vEnv
.
CleanUp
()
}
}
// 使用CleanupTestDatabase命令手动清理残余的`optimizer_xxx`数据库
// 为了保护当前正在评审的SQL,只清除前1小时前的残余
if
common
.
Config
.
CleanupTestDatabase
{
vEnv
.
CleanupTestDatabase
()
}
// 对指定的库表进行索引重复检查
// 对指定的库表进行索引重复检查
if
common
.
Config
.
ReportType
==
"duplicate-key-checker"
{
if
common
.
Config
.
ReportType
==
"duplicate-key-checker"
{
dupKeySuggest
:=
advisor
.
DuplicateKeyChecker
(
rEnv
)
dupKeySuggest
:=
advisor
.
DuplicateKeyChecker
(
rEnv
)
...
...
common/config.go
浏览文件 @
08cf841e
...
@@ -43,6 +43,7 @@ type Configration struct {
...
@@ -43,6 +43,7 @@ type Configration struct {
TestDSN
*
dsn
`yaml:"test-dsn"`
// 测试环境数据库配置
TestDSN
*
dsn
`yaml:"test-dsn"`
// 测试环境数据库配置
AllowOnlineAsTest
bool
`yaml:"allow-online-as-test"`
// 允许Online环境也可以当作Test环境
AllowOnlineAsTest
bool
`yaml:"allow-online-as-test"`
// 允许Online环境也可以当作Test环境
DropTestTemporary
bool
`yaml:"drop-test-temporary"`
// 是否清理Test环境产生的临时库表
DropTestTemporary
bool
`yaml:"drop-test-temporary"`
// 是否清理Test环境产生的临时库表
CleanupTestDatabase
bool
`yaml:"cleanup-test-database"`
// 清理残余的测试数据库(程序异常退出或未开启drop-test-temporary) issue #48
OnlySyntaxCheck
bool
`yaml:"only-syntax-check"`
// 只做语法检查不输出优化建议
OnlySyntaxCheck
bool
`yaml:"only-syntax-check"`
// 只做语法检查不输出优化建议
SamplingStatisticTarget
int
`yaml:"sampling-statistic-target"`
// 数据采样因子,对应postgres的default_statistics_target
SamplingStatisticTarget
int
`yaml:"sampling-statistic-target"`
// 数据采样因子,对应postgres的default_statistics_target
Sampling
bool
`yaml:"sampling"`
// 数据采样开关
Sampling
bool
`yaml:"sampling"`
// 数据采样开关
...
@@ -119,7 +120,6 @@ type Configration struct {
...
@@ -119,7 +120,6 @@ type Configration struct {
Verbose
bool
`yaml:"verbose"`
// verbose模式,会多输出一些信息
Verbose
bool
`yaml:"verbose"`
// verbose模式,会多输出一些信息
DryRun
bool
`yaml:"dry-run"`
// 是否在预演环境执行
DryRun
bool
`yaml:"dry-run"`
// 是否在预演环境执行
MaxPrettySQLLength
int
`yaml:"max-pretty-sql-length"`
// 超出该长度的SQL会转换成指纹输出
MaxPrettySQLLength
int
`yaml:"max-pretty-sql-length"`
// 超出该长度的SQL会转换成指纹输出
CleanTestDataBase
bool
`yaml:"clean-test-database"`
// 清理数据库 issue #48
}
}
// getDefaultLogOutput get default log-output by runtime.GOOS
// getDefaultLogOutput get default log-output by runtime.GOOS
...
@@ -146,6 +146,7 @@ var Config = &Configration{
...
@@ -146,6 +146,7 @@ var Config = &Configration{
},
},
AllowOnlineAsTest
:
false
,
AllowOnlineAsTest
:
false
,
DropTestTemporary
:
true
,
DropTestTemporary
:
true
,
CleanupTestDatabase
:
false
,
DryRun
:
true
,
DryRun
:
true
,
OnlySyntaxCheck
:
false
,
OnlySyntaxCheck
:
false
,
SamplingStatisticTarget
:
100
,
SamplingStatisticTarget
:
100
,
...
@@ -220,7 +221,6 @@ var Config = &Configration{
...
@@ -220,7 +221,6 @@ var Config = &Configration{
ListTestSqls
:
false
,
ListTestSqls
:
false
,
ListReportTypes
:
false
,
ListReportTypes
:
false
,
MaxPrettySQLLength
:
1024
,
MaxPrettySQLLength
:
1024
,
CleanTestDataBase
:
false
,
}
}
type
dsn
struct
{
type
dsn
struct
{
...
@@ -480,6 +480,7 @@ func readCmdFlags() error {
...
@@ -480,6 +480,7 @@ func readCmdFlags() error {
testDSN
:=
flag
.
String
(
"test-dsn"
,
FormatDSN
(
Config
.
TestDSN
),
"TestDSN, 测试环境数据库配置, username:password@ip:port/schema"
)
testDSN
:=
flag
.
String
(
"test-dsn"
,
FormatDSN
(
Config
.
TestDSN
),
"TestDSN, 测试环境数据库配置, username:password@ip:port/schema"
)
allowOnlineAsTest
:=
flag
.
Bool
(
"allow-online-as-test"
,
Config
.
AllowOnlineAsTest
,
"AllowOnlineAsTest, 允许线上环境也可以当作测试环境"
)
allowOnlineAsTest
:=
flag
.
Bool
(
"allow-online-as-test"
,
Config
.
AllowOnlineAsTest
,
"AllowOnlineAsTest, 允许线上环境也可以当作测试环境"
)
dropTestTemporary
:=
flag
.
Bool
(
"drop-test-temporary"
,
Config
.
DropTestTemporary
,
"DropTestTemporary, 是否清理测试环境产生的临时库表"
)
dropTestTemporary
:=
flag
.
Bool
(
"drop-test-temporary"
,
Config
.
DropTestTemporary
,
"DropTestTemporary, 是否清理测试环境产生的临时库表"
)
cleanupTestDatabase
:=
flag
.
Bool
(
"cleanup-test-database"
,
Config
.
CleanupTestDatabase
,
"单次运行清理历史1小时前残余的测试库。"
)
onlySyntaxCheck
:=
flag
.
Bool
(
"only-syntax-check"
,
Config
.
OnlySyntaxCheck
,
"OnlySyntaxCheck, 只做语法检查不输出优化建议"
)
onlySyntaxCheck
:=
flag
.
Bool
(
"only-syntax-check"
,
Config
.
OnlySyntaxCheck
,
"OnlySyntaxCheck, 只做语法检查不输出优化建议"
)
profiling
:=
flag
.
Bool
(
"profiling"
,
Config
.
Profiling
,
"Profiling, 开启数据采样的情况下在测试环境执行Profile"
)
profiling
:=
flag
.
Bool
(
"profiling"
,
Config
.
Profiling
,
"Profiling, 开启数据采样的情况下在测试环境执行Profile"
)
trace
:=
flag
.
Bool
(
"trace"
,
Config
.
Trace
,
"Trace, 开启数据采样的情况下在测试环境执行Trace"
)
trace
:=
flag
.
Bool
(
"trace"
,
Config
.
Trace
,
"Trace, 开启数据采样的情况下在测试环境执行Trace"
)
...
@@ -547,7 +548,6 @@ func readCmdFlags() error {
...
@@ -547,7 +548,6 @@ func readCmdFlags() error {
verbose
:=
flag
.
Bool
(
"verbose"
,
Config
.
Verbose
,
"Verbose"
)
verbose
:=
flag
.
Bool
(
"verbose"
,
Config
.
Verbose
,
"Verbose"
)
dryrun
:=
flag
.
Bool
(
"dry-run"
,
Config
.
DryRun
,
"是否在预演环境执行"
)
dryrun
:=
flag
.
Bool
(
"dry-run"
,
Config
.
DryRun
,
"是否在预演环境执行"
)
maxPrettySQLLength
:=
flag
.
Int
(
"max-pretty-sql-length"
,
Config
.
MaxPrettySQLLength
,
"MaxPrettySQLLength, 超出该长度的SQL会转换成指纹输出"
)
maxPrettySQLLength
:=
flag
.
Int
(
"max-pretty-sql-length"
,
Config
.
MaxPrettySQLLength
,
"MaxPrettySQLLength, 超出该长度的SQL会转换成指纹输出"
)
cleanTestDataBase
:=
flag
.
Bool
(
"clean-test-database"
,
Config
.
CleanTestDataBase
,
"由于程序异常退出,导致临时数据库的存在,可用此命令删除。"
)
// 一个不存在log-level,用于更新usage。
// 一个不存在log-level,用于更新usage。
// 因为vitess里面也用了flag,这些vitess的参数我们不需要关注
// 因为vitess里面也用了flag,这些vitess的参数我们不需要关注
if
!
Config
.
Verbose
&&
runtime
.
GOOS
!=
"windows"
{
if
!
Config
.
Verbose
&&
runtime
.
GOOS
!=
"windows"
{
...
@@ -566,6 +566,7 @@ func readCmdFlags() error {
...
@@ -566,6 +566,7 @@ func readCmdFlags() error {
Config
.
TestDSN
=
parseDSN
(
*
testDSN
,
Config
.
TestDSN
)
Config
.
TestDSN
=
parseDSN
(
*
testDSN
,
Config
.
TestDSN
)
Config
.
AllowOnlineAsTest
=
*
allowOnlineAsTest
Config
.
AllowOnlineAsTest
=
*
allowOnlineAsTest
Config
.
DropTestTemporary
=
*
dropTestTemporary
Config
.
DropTestTemporary
=
*
dropTestTemporary
Config
.
CleanupTestDatabase
=
*
cleanupTestDatabase
Config
.
OnlySyntaxCheck
=
*
onlySyntaxCheck
Config
.
OnlySyntaxCheck
=
*
onlySyntaxCheck
Config
.
Profiling
=
*
profiling
Config
.
Profiling
=
*
profiling
Config
.
Trace
=
*
trace
Config
.
Trace
=
*
trace
...
@@ -651,7 +652,6 @@ func readCmdFlags() error {
...
@@ -651,7 +652,6 @@ func readCmdFlags() error {
Config
.
DryRun
=
*
dryrun
Config
.
DryRun
=
*
dryrun
Config
.
MaxPrettySQLLength
=
*
maxPrettySQLLength
Config
.
MaxPrettySQLLength
=
*
maxPrettySQLLength
Config
.
MaxVarcharLength
=
*
maxVarcharLength
Config
.
MaxVarcharLength
=
*
maxVarcharLength
Config
.
CleanTestDataBase
=
*
cleanTestDataBase
if
*
ver
{
if
*
ver
{
version
()
version
()
...
...
env/env.go
浏览文件 @
08cf841e
...
@@ -150,35 +150,42 @@ func (ve VirtualEnv) CleanUp() bool {
...
@@ -150,35 +150,42 @@ func (ve VirtualEnv) CleanUp() bool {
return
true
return
true
}
}
// Clean
TestDataB
ase 清除一小时前的环境
// Clean
upTestDatab
ase 清除一小时前的环境
func
(
ve
*
VirtualEnv
)
Clean
TestDataB
ase
()
{
func
(
ve
*
VirtualEnv
)
Clean
upTestDatab
ase
()
{
common
.
Log
.
Debug
(
"Clean
TestDataB
ase ..."
)
common
.
Log
.
Debug
(
"Clean
upTestDatab
ase ..."
)
dbs
,
err
:=
ve
.
Query
(
"show databases like 'optimizer%'"
)
dbs
,
err
:=
ve
.
Query
(
"show databases like 'optimizer%'"
)
if
err
==
nil
{
if
err
==
nil
{
for
index
,
row
:=
range
dbs
.
Rows
{
for
_
,
row
:=
range
dbs
.
Rows
{
s
:=
strings
.
Split
(
row
.
Str
(
index
),
"_"
)
testDatabase
:=
row
.
Str
(
0
)
// test temporary database format `opimtizer_YYMMDDHHmmss_randomString(16)`
if
len
(
testDatabase
)
!=
39
{
common
.
Log
.
Debug
(
"CleanupTestDatabase by pass %s"
,
testDatabase
)
continue
}
s
:=
strings
.
Split
(
testDatabase
,
"_"
)
pastTime
,
err
:=
time
.
Parse
(
"060102150405"
,
s
[
1
])
pastTime
,
err
:=
time
.
Parse
(
"060102150405"
,
s
[
1
])
if
err
!=
nil
{
if
err
!=
nil
{
common
.
Log
.
Error
(
"Clean
TestDataB
ase compute pastTime Error: %s"
,
err
)
common
.
Log
.
Error
(
"Clean
upTestDatab
ase compute pastTime Error: %s"
,
err
)
}
}
nowTime
,
err
:=
time
.
Parse
(
"060102150405"
,
time
.
Now
()
.
Format
(
"060102150405"
))
nowTime
,
err
:=
time
.
Parse
(
"060102150405"
,
time
.
Now
()
.
Format
(
"060102150405"
))
if
err
!=
nil
{
if
err
!=
nil
{
common
.
Log
.
Error
(
"Clean
TestDataBase compute
nowTime Error: %s"
,
err
)
common
.
Log
.
Error
(
"Clean
upTestDatabase compute
nowTime Error: %s"
,
err
)
}
}
// TODO: 1 hour should be configable
subHour
:=
nowTime
.
Sub
(
pastTime
)
.
Hours
()
subHour
:=
nowTime
.
Sub
(
pastTime
)
.
Hours
()
if
subHour
>
1
{
if
subHour
>
1
{
_
,
err
:=
ve
.
Query
(
"drop database %s"
,
row
.
Str
(
index
)
)
_
,
err
:=
ve
.
Query
(
"drop database %s"
,
testDatabase
)
if
err
!=
nil
{
if
err
!=
nil
{
common
.
Log
.
Error
(
"Clean
TestDataB
ase failed Error: %s"
,
err
)
common
.
Log
.
Error
(
"Clean
upTestDatab
ase failed Error: %s"
,
err
)
}
}
common
.
Log
.
Debug
(
"CleanTestDataBase, done"
)
common
.
Log
.
Debug
(
"CleanupTestDatabase drop database %s"
,
testDatabase
)
}
else
{
common
.
Log
.
Debug
(
"CleanupTestDatabase by pass database %s, less than %d hours"
,
testDatabase
,
subHour
)
}
}
}
}
}
else
{
}
else
{
common
.
Log
.
Error
(
"Clean
TestDataB
ase failed Error:%s"
,
err
)
common
.
Log
.
Error
(
"Clean
upTestDatab
ase failed Error:%s"
,
err
)
}
}
}
}
// BuildVirtualEnv rEnv为SQL源环境,DB使用的信息从接口获取
// BuildVirtualEnv rEnv为SQL源环境,DB使用的信息从接口获取
...
...
env/env_test.go
浏览文件 @
08cf841e
...
@@ -113,9 +113,34 @@ func TestNewVirtualEnv(t *testing.T) {
...
@@ -113,9 +113,34 @@ func TestNewVirtualEnv(t *testing.T) {
},
t
.
Name
(),
update
)
},
t
.
Name
(),
update
)
}
}
func
TestCleanupTestDatabase
(
t
*
testing
.
T
)
{
vEnv
,
_
:=
BuildEnv
()
vEnv
.
Query
(
"drop database if exists optimizer_100000000000_xxxxxxxxxxxxxxxx"
)
_
,
err
:=
vEnv
.
Query
(
"create database optimizer_100000000000_xxxxxxxxxxxxxxxx"
)
if
err
!=
nil
{
t
.
Error
(
err
)
}
vEnv
.
CleanupTestDatabase
()
_
,
err
=
vEnv
.
Query
(
"drop database optimizer_100000000000_xxxxxxxxxxxxxxxx"
)
if
err
==
nil
{
t
.
Error
(
"optimizer_100000000000_xxxxxxxxxxxxxxxx exist, should be droped"
)
}
vEnv
.
Query
(
"drop database if exists optimizer_100000000000"
)
_
,
err
=
vEnv
.
Query
(
"create database optimizer_100000000000"
)
if
err
!=
nil
{
t
.
Error
(
err
)
}
vEnv
.
CleanupTestDatabase
()
_
,
err
=
vEnv
.
Query
(
"drop database optimizer_100000000000"
)
if
err
!=
nil
{
t
.
Error
(
"optimizer_100000000000 not exist, should not be droped"
)
}
}
func
TestGenTableColumns
(
t
*
testing
.
T
)
{
func
TestGenTableColumns
(
t
*
testing
.
T
)
{
vEnv
,
rEnv
:=
BuildEnv
()
vEnv
,
rEnv
:=
BuildEnv
()
vEnv
.
Clean
TestDataB
ase
()
vEnv
.
Clean
upTestDatab
ase
()
defer
vEnv
.
CleanUp
()
defer
vEnv
.
CleanUp
()
pretty
.
Println
(
common
.
Config
.
TestDSN
.
Disable
)
pretty
.
Println
(
common
.
Config
.
TestDSN
.
Disable
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录