diff --git a/advisor/heuristic.go b/advisor/heuristic.go index 82d33d90db2dc8f6c21eb9fa67a0c5e8f6c2c401..841441dcc86b9f2582441dd6ce9225d69934d3ab 100644 --- a/advisor/heuristic.go +++ b/advisor/heuristic.go @@ -2135,6 +2135,21 @@ func (q *Query4Audit) RuleNot() Rule { return rule } +// RuleInsertValues ARG.012 +func (q *Query4Audit) RuleInsertValues() Rule { + var rule = q.RuleOK() + switch s := q.Stmt.(type) { + case *sqlparser.Insert: + switch val := s.Rows.(type) { + case sqlparser.Values: + if len(val) > common.Config.MaxValueCount { + rule = HeuristicRules["ARG.012"] + } + } + } + return rule +} + // RuleUNIONUsage SUB.002 func (q *Query4Audit) RuleUNIONUsage() Rule { var rule = q.RuleOK() diff --git a/advisor/heuristic_test.go b/advisor/heuristic_test.go index 6d0f3f1a7c3c9305ab3b0d6656c945b1d331ff68..de57328c13ee88ff90d84da51e5ff6739e96ec54 100644 --- a/advisor/heuristic_test.go +++ b/advisor/heuristic_test.go @@ -1888,7 +1888,7 @@ func TestRuleHint(t *testing.T) { } // ARG.011 -func TestNot(t *testing.T) { +func TestRuleNot(t *testing.T) { common.Log.Debug("Entering function: %s", common.GetFunctionName()) sqls := [][]string{ { @@ -1925,6 +1925,46 @@ func TestNot(t *testing.T) { common.Log.Debug("Exiting function: %s", common.GetFunctionName()) } +// ARG.012 +func TestRuleInsertValues(t *testing.T) { + common.Log.Debug("Entering function: %s", common.GetFunctionName()) + sqls := [][]string{ + { + `INSERT INTO tb VALUES (1), (2)`, + `REPLACE INTO tb VALUES (1), (2)`, + }, + { + `INSERT INTO tb VALUES (1)`, + }, + } + oldMaxValueCount := common.Config.MaxValueCount + common.Config.MaxValueCount = 1 + for _, sql := range sqls[0] { + q, err := NewQuery4Audit(sql) + if err == nil { + rule := q.RuleInsertValues() + if rule.Item != "ARG.012" { + t.Error("Rule not match:", rule.Item, "Expect : ARG.012") + } + } else { + t.Error("sqlparser.Parse Error:", err) + } + } + for _, sql := range sqls[1] { + q, err := NewQuery4Audit(sql) + if err == nil { + rule := q.RuleInsertValues() + if rule.Item != "OK" { + t.Error("Rule not match:", rule.Item, "Expect : OK") + } + } else { + t.Error("sqlparser.Parse Error:", err) + } + } + common.Config.MaxValueCount = oldMaxValueCount + common.Log.Debug("Exiting function: %s", common.GetFunctionName()) +} + // SUB.002 func TestRuleUNIONUsage(t *testing.T) { common.Log.Debug("Entering function: %s", common.GetFunctionName()) diff --git a/advisor/rules.go b/advisor/rules.go index 957f22d6bc26dc43988d44f473d63d437a52f3c0..29074aaa3b7e2384dfb8bdeb507018865871fd66 100644 --- a/advisor/rules.go +++ b/advisor/rules.go @@ -264,6 +264,14 @@ func init() { Case: "select id from t where num not in(1,2,3);", Func: (*Query4Audit).RuleNot, }, + "ARG.012": { + Item: "ARG.012", + Severity: "L2", + Summary: "一次性 INSERT/REPLACE 的数据过多", + Content: "单条 INSERT/REPLACE 语句批量插入大量数据性能较差,甚至可能导致从库同步延迟。为了提升性能,减少批量写入数据对从库同步延时的影响,建议采用分批次插入的方法。", + Case: "INSERT INTO tb (a) VALUES (1), (2)", + Func: (*Query4Audit).RuleInsertValues, + }, "CLA.001": { Item: "CLA.001", Severity: "L4", @@ -280,7 +288,6 @@ func init() { Case: "select name from tbl where id < 1000 order by rand(number)", Func: (*Query4Audit).RuleOrderByRand, }, - "CLA.003": { Item: "CLA.003", Severity: "L2", diff --git a/advisor/testdata/TestListHeuristicRules.golden b/advisor/testdata/TestListHeuristicRules.golden index b104bb64cae8c6209483e94755b914f843578a66..4ce689ec0da235ab818193e5c00adae1c601e013 100644 --- a/advisor/testdata/TestListHeuristicRules.golden +++ b/advisor/testdata/TestListHeuristicRules.golden @@ -182,6 +182,16 @@ SELECT * FROM t1 USE INDEX (i1) ORDER BY a; ```sql select id from t where num not in(1,2,3); ``` +## 一次性 INSERT/REPLACE 的数据过多 + +* **Item**:ARG.012 +* **Severity**:L2 +* **Content**:单条 INSERT/REPLACE 语句批量插入大量数据性能较差,甚至可能导致从库同步延迟。为了提升性能,减少批量写入数据对从库同步延时的影响,建议采用分批次插入的方法。 +* **Case**: + +```sql +INSERT INTO tb (a) VALUES (1), (2) +``` ## 最外层 SELECT 未指定 WHERE 条件 * **Item**:CLA.001 diff --git a/advisor/testdata/TestMergeConflictHeuristicRules.golden b/advisor/testdata/TestMergeConflictHeuristicRules.golden index 444d2ff2597da2064ca03a1d7f91c7a246e1edf4..595b9a407c241424e0f9f9654e8bb2b356723346 100644 --- a/advisor/testdata/TestMergeConflictHeuristicRules.golden +++ b/advisor/testdata/TestMergeConflictHeuristicRules.golden @@ -15,6 +15,7 @@ advisor.Rule{Item:"ARG.008", Severity:"L1", Summary:"OR 查询索引列时请尽 advisor.Rule{Item:"ARG.009", Severity:"L1", Summary:"引号中的字符串开头或结尾包含空格", Content:"如果 VARCHAR 列的前后存在空格将可能引起逻辑问题,如在 MySQL 5.5中 'a' 和 'a ' 可能会在查询中被认为是相同的值。", Case:"SELECT 'abc '", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}} advisor.Rule{Item:"ARG.010", Severity:"L1", Summary:"不要使用 hint,如:sql_no_cache, force index, ignore key, straight join等", Content:"hint 是用来强制 SQL 按照某个执行计划来执行,但随着数据量变化我们无法保证自己当初的预判是正确的。", Case:"SELECT * FROM t1 USE INDEX (i1) ORDER BY a;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}} advisor.Rule{Item:"ARG.011", Severity:"L3", Summary:"不要使用负向查询,如:NOT IN/NOT LIKE", Content:"请尽量不要使用负向查询,这将导致全表扫描,对查询性能影响较大。", Case:"select id from t where num not in(1,2,3);", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}} +advisor.Rule{Item:"ARG.012", Severity:"L2", Summary:"一次性 INSERT/REPLACE 的数据过多", Content:"单条 INSERT/REPLACE 语句批量插入大量数据性能较差,甚至可能导致从库同步延迟。为了提升性能,减少批量写入数据对从库同步延时的影响,建议采用分批次插入的方法。", Case:"INSERT INTO tb (a) VALUES (1), (2)", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}} advisor.Rule{Item:"CLA.001", Severity:"L4", Summary:"最外层 SELECT 未指定 WHERE 条件", Content:"SELECT 语句没有 WHERE 子句,可能检查比预期更多的行(全表扫描)。对于 SELECT COUNT(*) 类型的请求如果不要求精度,建议使用 SHOW TABLE STATUS 或 EXPLAIN 替代。", Case:"select id from tbl", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}} advisor.Rule{Item:"CLA.002", Severity:"L3", Summary:"不建议使用 ORDER BY RAND()", Content:"ORDER BY RAND() 是从结果集中检索随机行的一种非常低效的方法,因为它会对整个结果进行排序并丢弃其大部分数据。", Case:"select name from tbl where id < 1000 order by rand(number)", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}} advisor.Rule{Item:"CLA.003", Severity:"L2", Summary:"不建议使用带 OFFSET 的LIMIT 查询", Content:"使用 LIMIT 和 OFFSET 对结果集分页的复杂度是 O(n^2),并且会随着数据增大而导致性能问题。采用“书签”扫描的方法实现分页效率更高。", Case:"select c1,c2 from tbl where name=xx order by number limit 1 offset 20", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}} diff --git a/common/config.go b/common/config.go index c0b4c7921a794166f7f6ea0338ff9cf070880f30..4f84116c407b4870c96943b1559912f4354ad480 100644 --- a/common/config.go +++ b/common/config.go @@ -101,6 +101,7 @@ type Configuration struct { TableAllowEngines []string `yaml:"table-allow-engines"` // Table 允许使用的 Engine MaxIdxCount int `yaml:"max-index-count"` // 单张表允许最多索引数 MaxColCount int `yaml:"max-column-count"` // 单张表允许最大列数 + MaxValueCount int `yaml:"max-value-count"` // INSERT/REPLACE 单次允许批量写入的行数 IdxPrefix string `yaml:"index-prefix"` // 普通索引建议使用的前缀 UkPrefix string `yaml:"unique-key-prefix"` // 唯一键建议使用的前缀 MaxSubqueryDepth int `yaml:"max-subquery-depth"` // 子查询最大尝试 @@ -180,6 +181,7 @@ var Config = &Configuration{ TableAllowEngines: []string{"innodb"}, MaxIdxCount: 10, MaxColCount: 40, + MaxValueCount: 100, MaxInCount: 10, IdxPrefix: "idx_", UkPrefix: "uk_", @@ -532,6 +534,7 @@ func readCmdFlags() error { tableAllowEngines := flag.String("table-allow-engines", strings.ToLower(strings.Join(Config.TableAllowEngines, ",")), "TableAllowEngines") maxIdxCount := flag.Int("max-index-count", Config.MaxIdxCount, "MaxIdxCount, 单表最大索引个数") maxColCount := flag.Int("max-column-count", Config.MaxColCount, "MaxColCount, 单表允许的最大列数") + maxValueCount := flag.Int("max-value-count", Config.MaxValueCount, "MaxValueCount, INSERT/REPLACE 单次批量写入允许的行数") idxPrefix := flag.String("index-prefix", Config.IdxPrefix, "IdxPrefix") ukPrefix := flag.String("unique-key-prefix", Config.UkPrefix, "UkPrefix") maxSubqueryDepth := flag.Int("max-subquery-depth", Config.MaxSubqueryDepth, "MaxSubqueryDepth") @@ -628,6 +631,7 @@ func readCmdFlags() error { Config.TableAllowEngines = strings.Split(strings.ToLower(*tableAllowEngines), ",") Config.MaxIdxCount = *maxIdxCount Config.MaxColCount = *maxColCount + Config.MaxValueCount = *maxValueCount Config.IdxPrefix = *idxPrefix Config.UkPrefix = *ukPrefix Config.MaxSubqueryDepth = *maxSubqueryDepth diff --git a/doc/heuristic.md b/doc/heuristic.md index b104bb64cae8c6209483e94755b914f843578a66..4ce689ec0da235ab818193e5c00adae1c601e013 100644 --- a/doc/heuristic.md +++ b/doc/heuristic.md @@ -182,6 +182,16 @@ SELECT * FROM t1 USE INDEX (i1) ORDER BY a; ```sql select id from t where num not in(1,2,3); ``` +## 一次性 INSERT/REPLACE 的数据过多 + +* **Item**:ARG.012 +* **Severity**:L2 +* **Content**:单条 INSERT/REPLACE 语句批量插入大量数据性能较差,甚至可能导致从库同步延迟。为了提升性能,减少批量写入数据对从库同步延时的影响,建议采用分批次插入的方法。 +* **Case**: + +```sql +INSERT INTO tb (a) VALUES (1), (2) +``` ## 最外层 SELECT 未指定 WHERE 条件 * **Item**:CLA.001