Merge branch 'master' of github.com:XiaoMi/soar

advisor/heuristic.go

@@ -2135,6 +2135,21 @@ func (q *Query4Audit) RuleNot() Rule {
 	return rule
 }
 
+// RuleInsertValues ARG.012
+func (q *Query4Audit) RuleInsertValues() Rule {
+	var rule = q.RuleOK()
+	switch s := q.Stmt.(type) {
+	case *sqlparser.Insert:
+		switch val := s.Rows.(type) {
+		case sqlparser.Values:
+			if len(val) > common.Config.MaxValueCount {
+				rule = HeuristicRules["ARG.012"]
+			}
+		}
+	}
+	return rule
+}
+
 // RuleUNIONUsage SUB.002
 func (q *Query4Audit) RuleUNIONUsage() Rule {
 	var rule = q.RuleOK()

advisor/heuristic_test.go

@@ -1888,7 +1888,7 @@ func TestRuleHint(t *testing.T) {
 }
 
 // ARG.011
-func TestNot(t *testing.T) {
+func TestRuleNot(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	sqls := [][]string{
 		{
@@ -1925,6 +1925,46 @@ func TestNot(t *testing.T) {
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
 
+// ARG.012
+func TestRuleInsertValues(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	sqls := [][]string{
+		{
+			`INSERT INTO tb VALUES (1), (2)`,
+			`REPLACE INTO tb VALUES (1), (2)`,
+		},
+		{
+			`INSERT INTO tb VALUES (1)`,
+		},
+	}
+	oldMaxValueCount := common.Config.MaxValueCount
+	common.Config.MaxValueCount = 1
+	for _, sql := range sqls[0] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleInsertValues()
+			if rule.Item != "ARG.012" {
+				t.Error("Rule not match:", rule.Item, "Expect : ARG.012")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+	for _, sql := range sqls[1] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleInsertValues()
+			if rule.Item != "OK" {
+				t.Error("Rule not match:", rule.Item, "Expect : OK")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+	common.Config.MaxValueCount = oldMaxValueCount
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}
+
 // SUB.002
 func TestRuleUNIONUsage(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())

advisor/rules.go

@@ -264,6 +264,14 @@ func init() {
 			Case:     "select id from t where num not in(1,2,3);",
 			Func:     (*Query4Audit).RuleNot,
 		},
+		"ARG.012": {
+			Item:     "ARG.012",
+			Severity: "L2",
+			Summary:  "一次性 INSERT/REPLACE 的数据过多",
+			Content:  "单条 INSERT/REPLACE 语句批量插入大量数据性能较差，甚至可能导致从库同步延迟。为了提升性能，减少批量写入数据对从库同步延时的影响，建议采用分批次插入的方法。",
+			Case:     "INSERT INTO tb (a) VALUES (1), (2)",
+			Func:     (*Query4Audit).RuleInsertValues,
+		},
 		"CLA.001": {
 			Item:     "CLA.001",
 			Severity: "L4",
@@ -280,7 +288,6 @@ func init() {
 			Case:     "select name from tbl where id < 1000 order by rand(number)",
 			Func:     (*Query4Audit).RuleOrderByRand,
 		},
-
 		"CLA.003": {
 			Item:     "CLA.003",
 			Severity: "L2",

advisor/testdata/TestListHeuristicRules.golden

@@ -182,6 +182,16 @@ SELECT * FROM t1 USE INDEX (i1) ORDER BY a;
 ```sql
 select id from t where num not in(1,2,3);
 ```
+## 一次性 INSERT/REPLACE 的数据过多
+
+* **Item**:ARG.012
+* **Severity**:L2
+* **Content**:单条 INSERT/REPLACE 语句批量插入大量数据性能较差，甚至可能导致从库同步延迟。为了提升性能，减少批量写入数据对从库同步延时的影响，建议采用分批次插入的方法。
+* **Case**:
+
+```sql
+INSERT INTO tb (a) VALUES (1), (2)
+```
 ## 最外层 SELECT 未指定 WHERE 条件
 
 * **Item**:CLA.001

advisor/testdata/TestMergeConflictHeuristicRules.golden

@@ -15,6 +15,7 @@ advisor.Rule{Item:"ARG.008", Severity:"L1", Summary:"OR 查询索引列时请尽
 advisor.Rule{Item:"ARG.009", Severity:"L1", Summary:"引号中的字符串开头或结尾包含空格", Content:"如果 VARCHAR 列的前后存在空格将可能引起逻辑问题，如在 MySQL 5.5中 'a' 和 'a ' 可能会在查询中被认为是相同的值。", Case:"SELECT 'abc '", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"ARG.010", Severity:"L1", Summary:"不要使用 hint，如：sql_no_cache, force index, ignore key, straight join等", Content:"hint 是用来强制 SQL 按照某个执行计划来执行，但随着数据量变化我们无法保证自己当初的预判是正确的。", Case:"SELECT * FROM t1 USE INDEX (i1) ORDER BY a;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"ARG.011", Severity:"L3", Summary:"不要使用负向查询，如：NOT IN/NOT LIKE", Content:"请尽量不要使用负向查询，这将导致全表扫描，对查询性能影响较大。", Case:"select id from t where num not in(1,2,3);", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
+advisor.Rule{Item:"ARG.012", Severity:"L2", Summary:"一次性 INSERT/REPLACE 的数据过多", Content:"单条 INSERT/REPLACE 语句批量插入大量数据性能较差，甚至可能导致从库同步延迟。为了提升性能，减少批量写入数据对从库同步延时的影响，建议采用分批次插入的方法。", Case:"INSERT INTO tb (a) VALUES (1), (2)", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"CLA.001", Severity:"L4", Summary:"最外层 SELECT 未指定 WHERE 条件", Content:"SELECT 语句没有 WHERE 子句，可能检查比预期更多的行(全表扫描)。对于 SELECT COUNT(*) 类型的请求如果不要求精度，建议使用 SHOW TABLE STATUS 或 EXPLAIN 替代。", Case:"select id from tbl", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"CLA.002", Severity:"L3", Summary:"不建议使用 ORDER BY RAND()", Content:"ORDER BY RAND() 是从结果集中检索随机行的一种非常低效的方法，因为它会对整个结果进行排序并丢弃其大部分数据。", Case:"select name from tbl where id < 1000 order by rand(number)", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"CLA.003", Severity:"L2", Summary:"不建议使用带 OFFSET 的LIMIT 查询", Content:"使用 LIMIT 和 OFFSET 对结果集分页的复杂度是 O(n^2)，并且会随着数据增大而导致性能问题。采用“书签”扫描的方法实现分页效率更高。", Case:"select c1,c2 from tbl where name=xx order by number limit 1 offset 20", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}

common/config.go

@@ -101,6 +101,7 @@ type Configuration struct {
 	TableAllowEngines    []string `yaml:"table-allow-engines"`       // Table 允许使用的 Engine
 	MaxIdxCount          int      `yaml:"max-index-count"`           // 单张表允许最多索引数
 	MaxColCount          int      `yaml:"max-column-count"`          // 单张表允许最大列数
+	MaxValueCount        int      `yaml:"max-value-count"`           // INSERT/REPLACE 单次允许批量写入的行数
 	IdxPrefix            string   `yaml:"index-prefix"`              // 普通索引建议使用的前缀
 	UkPrefix             string   `yaml:"unique-key-prefix"`         // 唯一键建议使用的前缀
 	MaxSubqueryDepth     int      `yaml:"max-subquery-depth"`        // 子查询最大尝试
@@ -180,6 +181,7 @@ var Config = &Configuration{
 	TableAllowEngines:    []string{"innodb"},
 	MaxIdxCount:          10,
 	MaxColCount:          40,
+	MaxValueCount:        100,
 	MaxInCount:           10,
 	IdxPrefix:            "idx_",
 	UkPrefix:             "uk_",
@@ -532,6 +534,7 @@ func readCmdFlags() error {
 	tableAllowEngines := flag.String("table-allow-engines", strings.ToLower(strings.Join(Config.TableAllowEngines, ",")), "TableAllowEngines")
 	maxIdxCount := flag.Int("max-index-count", Config.MaxIdxCount, "MaxIdxCount, 单表最大索引个数")
 	maxColCount := flag.Int("max-column-count", Config.MaxColCount, "MaxColCount, 单表允许的最大列数")
+	maxValueCount := flag.Int("max-value-count", Config.MaxValueCount, "MaxValueCount, INSERT/REPLACE 单次批量写入允许的行数")
 	idxPrefix := flag.String("index-prefix", Config.IdxPrefix, "IdxPrefix")
 	ukPrefix := flag.String("unique-key-prefix", Config.UkPrefix, "UkPrefix")
 	maxSubqueryDepth := flag.Int("max-subquery-depth", Config.MaxSubqueryDepth, "MaxSubqueryDepth")
@@ -628,6 +631,7 @@ func readCmdFlags() error {
 	Config.TableAllowEngines = strings.Split(strings.ToLower(*tableAllowEngines), ",")
 	Config.MaxIdxCount = *maxIdxCount
 	Config.MaxColCount = *maxColCount
+	Config.MaxValueCount = *maxValueCount
 	Config.IdxPrefix = *idxPrefix
 	Config.UkPrefix = *ukPrefix
 	Config.MaxSubqueryDepth = *maxSubqueryDepth

doc/heuristic.md

@@ -182,6 +182,16 @@ SELECT * FROM t1 USE INDEX (i1) ORDER BY a;
 ```sql
 select id from t where num not in(1,2,3);
 ```
+## 一次性 INSERT/REPLACE 的数据过多
+
+* **Item**:ARG.012
+* **Severity**:L2
+* **Content**:单条 INSERT/REPLACE 语句批量插入大量数据性能较差，甚至可能导致从库同步延迟。为了提升性能，减少批量写入数据对从库同步延时的影响，建议采用分批次插入的方法。
+* **Case**:
+
+```sql
+INSERT INTO tb (a) VALUES (1), (2)
+```
 ## 最外层 SELECT 未指定 WHERE 条件
 
 * **Item**:CLA.001