Skip to content

S
soar

Xiaomi / soar
10 个月 前同步成功

通知 373
Star 8512
Fork 1328
S
soar

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 5619dd32 编写于 作者: martianzhang's avatar martianzhang
浏览文件
操作

code format add some test case

上级 5c730e2b
隐藏空白更改
内联 并排
Showing with 227 addition and 245 deletion
advisor/explainer.go
浏览文件 @ 5619dd32
......@@ -32,20 +32,13 @@ var explainRules map[string]Rule
// [table_name]"suggest text"
var tablesSuggests map[string][]string
/*
var explainIgnoreTables = []string{
"dual",
"",
}
*/
// explain建议的形式
// Item: EXP.XXX
// Severity: L[0-8]
// Summary: full table scan, not use index, full index scan...
// Content: XX TABLE xxx
//
// checkExplainSelectType
func checkExplainSelectType(exp *database.ExplainInfo) {
// 判断是否跳过不检查
if len(common.Config.ExplainWarnSelectType) == 1 {
......@@ -70,7 +63,7 @@ func checkExplainSelectType(exp *database.ExplainInfo) {
}
}
// 用户可以设置AccessType的建议级别，匹配到的查询会给出建议
// checkExplainAccessType 用户可以设置AccessType的建议级别，匹配到的查询会给出建议
func checkExplainAccessType(exp *database.ExplainInfo) {
// 判断是否跳过不检查
if len(common.Config.ExplainWarnAccessType) == 1 {
......@@ -95,43 +88,28 @@ func checkExplainAccessType(exp *database.ExplainInfo) {
}
}
// TODO:
/*
// TODO:
func checkExplainPossibleKeys(exp *database.ExplainInfo) {
// 判断是否跳过不检查
if common.Config.ExplainMinPossibleKeys == 0 {
return
}
rows := exp.ExplainRows
if exp.ExplainFormat == database.JSONFormatExplain {
// JSON形式遍历分析不方便，转成Row格式统一处理
rows = database.ConvertExplainJSON2Row(exp.ExplainJSON)
}
for _, row := range rows {
if len(row.PossibleKeys) < common.Config.ExplainMinPossibleKeys {
tablesSuggests[row.TableName] = append(tablesSuggests[row.TableName], fmt.Sprintf("PossibleKeys:%d < %d",
len(row.PossibleKeys), common.Config.ExplainMinPossibleKeys))
}
}
}
*/
// TODO:
/*
func checkExplainKeyLen(exp *database.ExplainInfo) {
}
*/
// TODO:
/*
func checkExplainKey(exp *database.ExplainInfo) {
// 小于最小使用试用key数量
//return intval($explainResult) < intval($userCond);
//explain-min-keys int
// 小于最小使用试用key数量
//return intval($explainResult) < intval($userCond);
//explain-min-keys int
}
func checkExplainExtra(exp *database.ExplainInfo) {
// 包含用户配置的逗号分隔关键词之一则提醒
// return self::contains($explainResult, $userCond);
// explain-warn-extra []string
}
*/
// checkExplainRef ...
func checkExplainRef(exp *database.ExplainInfo) {
rows := exp.ExplainRows
if exp.ExplainFormat == database.JSONFormatExplain {
......@@ -148,6 +126,7 @@ func checkExplainRef(exp *database.ExplainInfo) {
}
}
// checkExplainRows ...
func checkExplainRows(exp *database.ExplainInfo) {
// 判断是否跳过不检查
if common.Config.ExplainMaxRows <= 0 {
......@@ -167,15 +146,7 @@ func checkExplainRows(exp *database.ExplainInfo) {
}
}
// TODO:
/*
func checkExplainExtra(exp *database.ExplainInfo) {
// 包含用户配置的逗号分隔关键词之一则提醒
// return self::contains($explainResult, $userCond);
// explain-warn-extra []string
}
*/
// checkExplainFiltered ...
func checkExplainFiltered(exp *database.ExplainInfo) {
// 判断是否跳过不检查
if common.Config.ExplainMaxFiltered <= 0.001 {
......@@ -235,30 +206,7 @@ func ExplainAdvisor(exp *database.ExplainInfo) map[string]Rule {
Func: (*Query4Audit).RuleOK,
}
}
/*
for t, s := range tablesSuggests {
// 检查explain对应的表是否需要跳过，如dual,空表等
ig := false
for _, ti := range explainIgnoreTables {
if ti == t {
ig = true
}
}
if ig {
continue
}
ruleId := fmt.Sprintf("EXP.%03d", explainRuleId+1)
explainRuleId = explainRuleId + 1
explainRules[ruleId] = Rule{
Item: ruleId,
Severity: "L0",
Summary: fmt.Sprintf("表 `%s` 查询效率不高", t),
Content: fmt.Sprint("原因：", strings.Join(s, ",")),
Case: "",
Func: (*Query4Audit).RuleOK,
}
}
*/
// TODO: 检查explain对应的表是否需要跳过，如dual,空表等
return explainRules
}
......
advisor/index.go
浏览文件 @ 5619dd32
......@@ -665,7 +665,7 @@ func (idxAdv *IndexAdvisor) buildIndex(idxList map[string]map[string][]*common.C
continue
}
idxName := "idx_" + strings.Join(colNames, "_")
idxName := common.Config.IdxPrefix + strings.Join(colNames, "_")
// 索引名称最大长度64
if len(idxName) > IndexNameMaxLength {
......@@ -699,7 +699,7 @@ func (idxAdv *IndexAdvisor) buildIndexWithNoEnv(indexList map[string]map[string]
common.Log.Warn("can not get the meta info of column '%s'", col.Name)
continue
}
idxName := "idx_" + col.Name
idxName := common.Config.IdxPrefix + col.Name
// 库、表、列名需要用反撇转义
alterSQL := fmt.Sprintf("alter table `%s`.`%s` add index `%s` (`%s`)", idxAdv.vEnv.RealDB(col.DB), col.Table, idxName, col.Name)
if col.DB == "" {
......
advisor/rules_test.go
浏览文件 @ 5619dd32
......@@ -48,9 +48,15 @@ func TestListHeuristicRules(t *testing.T) {
func TestInBlackList(t *testing.T) {
common.Log.Debug("Entering function: %s", common.GetFunctionName())
sqls := []string{
"select",
"select 1",
}
common.BlackList = []string{"select"}
if !InBlackList("select 1") {
t.Error("should be true")
for _, sql := range sqls {
if !InBlackList(sql) {
t.Error("should be true")
}
}
common.Log.Debug("Exiting function: %s", common.GetFunctionName())
}
......
ast/rewrite.go
浏览文件 @ 5619dd32
......@@ -40,180 +40,184 @@ type Rule struct {
}
// RewriteRules SQL重写规则，注意这个规则是有序的，先后顺序不能乱
var RewriteRules = []Rule{
{
Name: "dml2select",
Description: "将数据库更新请求转换为只读查询请求，便于执行EXPLAIN",
Original: "DELETE FROM film WHERE length > 100",
Suggest: "select * from film where length > 100",
Func: (*Rewrite).RewriteDML2Select,
},
{
Name: "star2columns",
Description: "为SELECT *补全表的列信息",
Original: "SELECT * FROM film",
Suggest: "select film.film_id, film.title from film",
Func: (*Rewrite).RewriteStar2Columns,
},
{
Name: "insertcolumns",
Description: "为INSERT补全表的列信息",
Original: "insert into film values(1,2,3,4,5)",
Suggest: "insert into film(film_id, title, description, release_year, language_id) values (1, 2, 3, 4, 5)",
Func: (*Rewrite).RewriteInsertColumns,
},
{
Name: "having",
Description: "将查询的 HAVING 子句改写为 WHERE 中的查询条件",
Original: "SELECT state, COUNT(*) FROM Drivers GROUP BY state HAVING state IN ('GA', 'TX') ORDER BY state",
Suggest: "select state, COUNT(*) from Drivers where state in ('GA', 'TX') group by state order by state asc",
Func: (*Rewrite).RewriteHaving,
},
{
Name: "orderbynull",
Description: "如果 GROUP BY 语句不指定 ORDER BY 条件会导致无谓的排序产生，如果不需要排序建议添加 ORDER BY NULL",
Original: "SELECT sum(col1) FROM tbl GROUP BY col",
Suggest: "select sum(col1) from tbl group by col order by null",
Func: (*Rewrite).RewriteAddOrderByNull,
},
{
Name: "unionall",
Description: "可以接受重复的时间，使用 UNION ALL 替代 UNION 以提高查询效率",
Original: "select country_id from city union select country_id from country",
Suggest: "select country_id from city union all select country_id from country",
Func: (*Rewrite).RewriteUnionAll,
},
{
Name: "or2in",
Description: "将同一列不同条件的 OR 查询转写为 IN 查询",
Original: "select country_id from city where col1 = 1 or (col2 = 1 or col2 = 2 ) or col1 = 3;",
Suggest: "select country_id from city where (col2 in (1, 2)) or col1 in (1, 3);",
Func: (*Rewrite).RewriteOr2In,
},
{
Name: "innull",
Description: "如果 IN 条件中可能有 NULL 值而又想匹配 NULL 值时，建议添加OR col IS NULL",
Original: "暂不支持",
Suggest: "暂不支持",
Func: (*Rewrite).RewriteInNull,
},
// 把所有跟 or 相关的重写完之后才进行 or 转 union 的重写
{
Name: "or2union",
Description: "将不同列的 OR 查询转为 UNION 查询，建议结合 unionall 重写策略一起使用",
Original: "暂不支持",
Suggest: "暂不支持",
Func: (*Rewrite).RewriteOr2Union,
},
{
Name: "dmlorderby",
Description: "删除 DML 更新操作中无意义的 ORDER BY",
Original: "DELETE FROM tbl WHERE col1=1 ORDER BY col",
Suggest: "delete from tbl where col1 = 1",
Func: (*Rewrite).RewriteRemoveDMLOrderBy,
},
/*
var RewriteRules []Rule
func init() {
RewriteRules = []Rule{
{
Name: "dml2select",
Description: "将数据库更新请求转换为只读查询请求，便于执行EXPLAIN",
Original: "DELETE FROM film WHERE length > 100",
Suggest: "select * from film where length > 100",
Func: (*Rewrite).RewriteDML2Select,
},
{
Name: "star2columns",
Description: "为SELECT *补全表的列信息",
Original: "SELECT * FROM film",
Suggest: "select film.film_id, film.title from film",
Func: (*Rewrite).RewriteStar2Columns,
},
{
Name: "insertcolumns",
Description: "为INSERT补全表的列信息",
Original: "insert into film values(1,2,3,4,5)",
Suggest: "insert into film(film_id, title, description, release_year, language_id) values (1, 2, 3, 4, 5)",
Func: (*Rewrite).RewriteInsertColumns,
},
{
Name: "having",
Description: "将查询的 HAVING 子句改写为 WHERE 中的查询条件",
Original: "SELECT state, COUNT(*) FROM Drivers GROUP BY state HAVING state IN ('GA', 'TX') ORDER BY state",
Suggest: "select state, COUNT(*) from Drivers where state in ('GA', 'TX') group by state order by state asc",
Func: (*Rewrite).RewriteHaving,
},
{
Name: "orderbynull",
Description: "如果 GROUP BY 语句不指定 ORDER BY 条件会导致无谓的排序产生，如果不需要排序建议添加 ORDER BY NULL",
Original: "SELECT sum(col1) FROM tbl GROUP BY col",
Suggest: "select sum(col1) from tbl group by col order by null",
Func: (*Rewrite).RewriteAddOrderByNull,
},
{
Name: "unionall",
Description: "可以接受重复的时间，使用 UNION ALL 替代 UNION 以提高查询效率",
Original: "select country_id from city union select country_id from country",
Suggest: "select country_id from city union all select country_id from country",
Func: (*Rewrite).RewriteUnionAll,
},
{
Name: "or2in",
Description: "将同一列不同条件的 OR 查询转写为 IN 查询",
Original: "select country_id from city where col1 = 1 or (col2 = 1 or col2 = 2 ) or col1 = 3;",
Suggest: "select country_id from city where (col2 in (1, 2)) or col1 in (1, 3);",
Func: (*Rewrite).RewriteOr2In,
},
{
Name: "innull",
Description: "如果 IN 条件中可能有 NULL 值而又想匹配 NULL 值时，建议添加OR col IS NULL",
Original: "暂不支持",
Suggest: "暂不支持",
Func: (*Rewrite).RewriteInNull,
},
// 把所有跟 or 相关的重写完之后才进行 or 转 union 的重写
{
Name: "or2union",
Description: "将不同列的 OR 查询转为 UNION 查询，建议结合 unionall 重写策略一起使用",
Original: "暂不支持",
Suggest: "暂不支持",
Func: (*Rewrite).RewriteOr2Union,
},
{
Name: "dmlorderby",
Description: "删除 DML 更新操作中无意义的 ORDER BY",
Original: "DELETE FROM tbl WHERE col1=1 ORDER BY col",
Suggest: "delete from tbl where col1 = 1",
Func: (*Rewrite).RewriteRemoveDMLOrderBy,
},
/*
{
Name: "groupbyconst",
Description: "删除无意义的GROUP BY常量",
Original: "SELECT sum(col1) FROM tbl GROUP BY 1;",
Suggest: "select sum(col1) from tbl",
Func: (*Rewrite).RewriteGroupByConst,
},
*/
{
Name: "sub2join",
Description: "将子查询转换为JOIN查询",
Original: "暂不支持",
Suggest: "暂不支持",
Func: (*Rewrite).RewriteSubQuery2Join,
},
{
Name: "groupbyconst",
Description: "删除无意义的GROUP BY常量",
Name: "join2sub",
Description: "将JOIN查询转换为子查询",
Original: "暂不支持",
Suggest: "暂不支持",
Func: (*Rewrite).RewriteJoin2SubQuery,
},
{
Name: "distinctstar",
Description: "DISTINCT *对有主键的表没有意义，可以将DISTINCT删掉",
Original: "SELECT DISTINCT * FROM film;",
Suggest: "SELECT * FROM film",
Func: (*Rewrite).RewriteDistinctStar,
},
{
Name: "standard",
Description: "SQL标准化，如：关键字转换为小写",
Original: "SELECT sum(col1) FROM tbl GROUP BY 1;",
Suggest: "select sum(col1) from tbl",
Func: (*Rewrite).RewriteGroupByConst,
Suggest: "select sum(col1) from tbl group by 1",
Func: (*Rewrite).RewriteStandard,
},
{
Name: "mergealter",
Description: "合并同一张表的多条ALTER语句",
Original: "ALTER TABLE t2 DROP COLUMN c;ALTER TABLE t2 DROP COLUMN d;",
Suggest: "ALTER TABLE t2 DROP COLUMN c, DROP COLUMN d;",
},
{
Name: "alwaystrue",
Description: "删除无用的恒真判断条件",
Original: "SELECT count(col) FROM tbl where 'a'= 'a' or ('b' = 'b' and a = 'b');",
Suggest: "select count(col) from tbl where (a = 'b');",
Func: (*Rewrite).RewriteAlwaysTrue,
},
{
Name: "countstar",
Description: "不建议使用COUNT(col)或COUNT(常量)，建议改写为COUNT(*)",
Original: "SELECT count(col) FROM tbl GROUP BY 1;",
Suggest: "SELECT count(*) FROM tbl GROUP BY 1;",
Func: (*Rewrite).RewriteCountStar,
},
{
Name: "innodb",
Description: "建表时建议使用InnoDB引擎，非 InnoDB 引擎表自动转 InnoDB",
Original: "CREATE TABLE t1(id bigint(20) NOT NULL AUTO_INCREMENT);",
Suggest: "create table t1 (\n\tid bigint(20) not null auto_increment\n) ENGINE=InnoDB;",
Func: (*Rewrite).RewriteInnoDB,
},
*/
{
Name: "sub2join",
Description: "将子查询转换为JOIN查询",
Original: "暂不支持",
Suggest: "暂不支持",
Func: (*Rewrite).RewriteSubQuery2Join,
},
{
Name: "join2sub",
Description: "将JOIN查询转换为子查询",
Original: "暂不支持",
Suggest: "暂不支持",
Func: (*Rewrite).RewriteJoin2SubQuery,
},
{
Name: "distinctstar",
Description: "DISTINCT *对有主键的表没有意义，可以将DISTINCT删掉",
Original: "SELECT DISTINCT * FROM film;",
Suggest: "SELECT * FROM film",
Func: (*Rewrite).RewriteDistinctStar,
},
{
Name: "standard",
Description: "SQL标准化，如：关键字转换为小写",
Original: "SELECT sum(col1) FROM tbl GROUP BY 1;",
Suggest: "select sum(col1) from tbl group by 1",
Func: (*Rewrite).RewriteStandard,
},
{
Name: "mergealter",
Description: "合并同一张表的多条ALTER语句",
Original: "ALTER TABLE t2 DROP COLUMN c;ALTER TABLE t2 DROP COLUMN d;",
Suggest: "ALTER TABLE t2 DROP COLUMN c, DROP COLUMN d;",
},
{
Name: "alwaystrue",
Description: "删除无用的恒真判断条件",
Original: "SELECT count(col) FROM tbl where 'a'= 'a' or ('b' = 'b' and a = 'b');",
Suggest: "select count(col) from tbl where (a = 'b');",
Func: (*Rewrite).RewriteAlwaysTrue,
},
{
Name: "countstar",
Description: "不建议使用COUNT(col)或COUNT(常量)，建议改写为COUNT(*)",
Original: "SELECT count(col) FROM tbl GROUP BY 1;",
Suggest: "SELECT count(*) FROM tbl GROUP BY 1;",
Func: (*Rewrite).RewriteCountStar,
},
{
Name: "innodb",
Description: "建表时建议使用InnoDB引擎，非 InnoDB 引擎表自动转 InnoDB",
Original: "CREATE TABLE t1(id bigint(20) NOT NULL AUTO_INCREMENT);",
Suggest: "create table t1 (\n\tid bigint(20) not null auto_increment\n) ENGINE=InnoDB;",
Func: (*Rewrite).RewriteInnoDB,
},
{
Name: "autoincrement",
Description: "将autoincrement初始化为1",
Original: "CREATE TABLE t1(id bigint(20) NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=123802;",
Suggest: "create table t1(id bigint(20) not null auto_increment) ENGINE=InnoDB auto_increment=1;",
Func: (*Rewrite).RewriteAutoIncrement,
},
{
Name: "intwidth",
Description: "整型数据类型修改默认显示宽度",
Original: "create table t1 (id int(20) not null auto_increment) ENGINE=InnoDB;",
Suggest: "create table t1 (id int(10) not null auto_increment) ENGINE=InnoDB;",
Func: (*Rewrite).RewriteIntWidth,
},
{
Name: "truncate",
Description: "不带 WHERE 条件的 DELETE 操作建议修改为 TRUNCATE",
Original: "DELETE FROM tbl",
Suggest: "truncate table tbl",
Func: (*Rewrite).RewriteTruncate,
},
{
Name: "rmparenthesis",
Description: "去除没有意义的括号",
Original: "select col from table where (col = 1);",
Suggest: "select col from table where col = 1;",
Func: (*Rewrite).RewriteRmParenthesis,
},
// delimiter要放在最后，不然补不上
{
Name: "delimiter",
Description: "补全DELIMITER",
Original: "use sakila",
Suggest: "use sakila;",
Func: (*Rewrite).RewriteDelimiter,
},
// TODO in to exists
// TODO exists to in
{
Name: "autoincrement",
Description: "将autoincrement初始化为1",
Original: "CREATE TABLE t1(id bigint(20) NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=123802;",
Suggest: "create table t1(id bigint(20) not null auto_increment) ENGINE=InnoDB auto_increment=1;",
Func: (*Rewrite).RewriteAutoIncrement,
},
{
Name: "intwidth",
Description: "整型数据类型修改默认显示宽度",
Original: "create table t1 (id int(20) not null auto_increment) ENGINE=InnoDB;",
Suggest: "create table t1 (id int(10) not null auto_increment) ENGINE=InnoDB;",
Func: (*Rewrite).RewriteIntWidth,
},
{
Name: "truncate",
Description: "不带 WHERE 条件的 DELETE 操作建议修改为 TRUNCATE",
Original: "DELETE FROM tbl",
Suggest: "truncate table tbl",
Func: (*Rewrite).RewriteTruncate,
},
{
Name: "rmparenthesis",
Description: "去除没有意义的括号",
Original: "select col from table where (col = 1);",
Suggest: "select col from table where col = 1;",
Func: (*Rewrite).RewriteRmParenthesis,
},
// delimiter要放在最后，不然补不上
{
Name: "delimiter",
Description: "补全DELIMITER",
Original: "use sakila",
Suggest: "use sakila;",
Func: (*Rewrite).RewriteDelimiter,
},
// TODO in to exists
// TODO exists to in
}
}
// ListRewriteRules 打印SQL重写规则
......@@ -842,7 +846,7 @@ func (rw *Rewrite) RewriteAddOrderByNull() *Rewrite {
}
// RewriteOr2Union or2union: 将 OR 查询转写为 UNION ALL TODO: 暂无对应 HeuristicRules
// https://sqlperformance.com/2014/09/sql-plan/rewriting-queries-improve-performance
// TODO: https://sqlperformance.com/2014/09/sql-plan/rewriting-queries-improve-performance
func (rw *Rewrite) RewriteOr2Union() *Rewrite {
return rw
}
......
cmd/soar/soar_test.go
浏览文件 @ 5619dd32
......@@ -18,6 +18,7 @@ package main
import (
"flag"
"fmt"
"testing"
"github.com/XiaoMi/soar/common"
......@@ -42,32 +43,42 @@ func TestMain(m *testing.M) {
}
func Test_Main(_ *testing.T) {
common.Log.Debug("Entering function: %s", common.GetFunctionName())
common.Config.OnlineDSN.Disable = true
common.Config.LogLevel = 0
common.Config.Query = "select * from film;alter table city add index idx_country_id(country_id);"
main()
common.Log.Debug("Exiting function: %s", common.GetFunctionName())
}
func Test_Main_More(_ *testing.T) {
common.Log.Debug("Entering function: %s", common.GetFunctionName())
common.Config.LogLevel = 0
common.Config.Profiling = true
common.Config.Explain = true
common.Config.Query = "select * from film where country_id = 1;use sakila;alter table city add index idx_country_id(country_id);"
orgRerportType := common.Config.ReportType
for _, typ := range []string{
"json", "html", "markdown", "fingerprint", "compress", "pretty", "rewrite",
"ast", "tiast", "ast-json", "tiast-json", "tokenize",
} {
common.Config.ReportType = typ
main()
}
common.Config.ReportType = orgRerportType
common.Log.Debug("Exiting function: %s", common.GetFunctionName())
}
func Test_Main_checkConfig(t *testing.T) {
common.Log.Debug("Entering function: %s", common.GetFunctionName())
if checkConfig() != 0 {
t.Error("checkConfig error")
}
common.Log.Debug("Exiting function: %s", common.GetFunctionName())
}
func Test_Main_initQuery(t *testing.T) {
common.Log.Debug("Entering function: %s", common.GetFunctionName())
// direct query
query := initQuery("select 1")
if query != "select 1" {
......@@ -79,4 +90,17 @@ func Test_Main_initQuery(t *testing.T) {
// TODO: read from stdin
// initQuery("")
common.Log.Debug("Exiting function: %s", common.GetFunctionName())
}
func Test_Main_reportTool(t *testing.T) {
common.Log.Debug("Entering function: %s", common.GetFunctionName())
orgRerportType := common.Config.ReportType
types := []string{"html", "md2html", "explain-digest", "chardet", "remove-comment"}
for _, tp := range types {
common.Config.ReportType = tp
fmt.Println(reportTool(tp, []byte{}))
}
common.Config.ReportType = orgRerportType
common.Log.Debug("Exiting function: %s", common.GetFunctionName())
}
common/config.go
浏览文件 @ 5619dd32
......@@ -294,7 +294,7 @@ func parseDSN(odbc string, d *Dsn) *Dsn {
userInfo = res[1]
hostInfo = res[2]
query = res[4]
} else if res := regexp.MustCompile(`^(.*?)($|\?)(.*)`).FindStringSubmatch(odbc); len(res) > 3 {
} else if res := regexp.MustCompile(`^(.*?)($|\?)(.*)`).FindStringSubmatch(odbc); len(res) > 3 {
// hostInfo
hostInfo = res[1]
query = res[3]
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或