From b1c977b1a169763f0878cbb114b1e7575ef827f4 Mon Sep 17 00:00:00 2001 From: Leon Zhang Date: Tue, 18 Dec 2018 16:00:06 +0800 Subject: [PATCH] try to fix #169 RES.009 not consider about "and", "or"... Operator in BinaryOperationExpr --- CHANGES.md | 32 +++-- advisor/heuristic.go | 12 +- advisor/heuristic_test.go | 4 + common/config.go | 2 +- .../pingcap/parser/ast/expressions.go | 52 ++++++++- vendor/vendor.json | 110 +++++++++--------- 6 files changed, 138 insertions(+), 74 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9472ca8..f3701e2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,11 @@ # CHANGELOG ## 2018-12 + - DOING: english translation ## 2018-11 + - add all third-party lib into vendor - support `-report-type chardet` - add more heuristic rules: TBL.008, KEY.010, ARG.012, KWR.004 @@ -21,33 +23,36 @@ - fix #112 multi-line comment will cause line counter error, when -report-type=lint - fix #110 remove bom before auditing - fix #104 case insensitive regex @ CLA.009 -- fix #87 RuleImplicitConversion value type mistach check bug +- fix #87 RuleImplicitConversion value type mismatch check bug - fix #38 always true where condition check - abandon stdin terminal interactive mod, which may seems like hangup ## 2018-10 -- Fix SplitStatement mulitstatement eof bug #66 + +- Fix SplitStatement multistatement eof bug #66 - Fix pretty func hangup issue #47 - Fix some foolish code spell error - Use travis for CI -- Fix Go 1.8 defapth GOPATH compatible issue BUG #5 +- Fix Go 1.8 default GOPATH compatible issue BUG #5 - 2018-10-20 开源先锋日(OSCAR)对外正式开源发布代码 ## 2018-09 + - 修复多个启发式建议不准确BUG，优化部分建议文案使得建议更清晰 -- 基于TiDB Parser完善多个DDL类型语句的建议 +- 基于 TiDB Parser 完善多个 DDL 类型语句的建议 - 新增lint report-type类型，支持Vim Plugin优化建议输出 - 更新整理项目文档，开源准备 - 2018-09-21 Gdevops SOAR首次对外进行技术分享宣传 ## 2018-08 -- 利用docker临时容器进行daily测试 + +- 利用 docker 临时容器进行 daily 测试 - 添加main_test全功能回归测试 - 修复在测试中发现的问题 -- mymysql合并MySQL8.0相关PR，修改vendor依赖 +- mymysql 合并 MySQL8.0 相关PR，修改vendor依赖 - 改善HeuristicRule中的文案 - 持续集成Vitess Parser的改进 -- NewQuery4Audit结构体中引入TiDB Parser +- NewQuery4Audit 结构体中引入 TiDB Parser - 通过TiAST完成大量与 DDL 相关的TODO - 修改heuristic rules检查的返回值，提升拓展性 - 建议中引入Position，用于表示建议产生于SQL的位置 @@ -58,22 +63,25 @@ - 优化 doc 文档 ## 2018-07 + - 补充文档，添加项目LOGO - 改善代码质量提升测试覆盖度 - mymysql升级，支持MySQL 8.0 - 提供remove-comment小工具 - 提供索引重复检查小工具 -- HeuristicRule新增RuleSpaceAfterDot +- HeuristicRule 新增 RuleSpaceAfterDot - 支持字符集和Collation不相同时的隐式数据类型转换的检查 ## 2018-06 + - 支持更多的SQL Rewrite规则 - 添加SQL执行超时限制 - 索引优化建议支持对约束的检查 -- 修复数据采样中null值处理不正确的问题 -- Explain支持last_query_cost +- 修复数据采样中 NULL 值处理不正确的问题 +- Explain 支持 last_query_cost ## 2018-05 + - 添加数据采样功能 - 添加语句执行安全检查 - 支持DDL语法检查 @@ -84,6 +92,7 @@ - 支持SQL Pretty输出 ## 2018-04 + - 支持语法检查 - 支持测试环境 - 支持MySQL原数据的获取 @@ -93,7 +102,8 @@ - 引入配置文件 ## 2018-03 + - 基本架构设计 - 添加大量底层函数用于处理AST -- 添加Insert、Delete、Update转写成Select的基本函数 +- 添加Insert、Delete、Update 转写成 Select 的基本函数 - 支持MySQL Explain信息输出 diff --git a/advisor/heuristic.go b/advisor/heuristic.go index 83f2747..441afec 100644 --- a/advisor/heuristic.go +++ b/advisor/heuristic.go @@ -1319,7 +1319,9 @@ func (q *Query4Audit) RuleMultiCompare() Rule { case *tidb.BinaryOperationExpr: switch where.L.(type) { case *tidb.BinaryOperationExpr: - rule = HeuristicRules["RES.009"] + if where.Op.String() == "eq" { + rule = HeuristicRules["RES.009"] + } } } case *tidb.UpdateStmt: @@ -1327,7 +1329,9 @@ func (q *Query4Audit) RuleMultiCompare() Rule { case *tidb.BinaryOperationExpr: switch where.L.(type) { case *tidb.BinaryOperationExpr: - rule = HeuristicRules["RES.009"] + if where.Op.String() == "eq" { + rule = HeuristicRules["RES.009"] + } } } case *tidb.DeleteStmt: @@ -1335,7 +1339,9 @@ func (q *Query4Audit) RuleMultiCompare() Rule { case *tidb.BinaryOperationExpr: switch where.L.(type) { case *tidb.BinaryOperationExpr: - rule = HeuristicRules["RES.009"] + if where.Op.String() == "eq" { + rule = HeuristicRules["RES.009"] + } } } } diff --git a/advisor/heuristic_test.go b/advisor/heuristic_test.go index 889aa5e..d313de3 100644 --- a/advisor/heuristic_test.go +++ b/advisor/heuristic_test.go @@ -952,6 +952,10 @@ func TestRuleMultiCompare(t *testing.T) { }, { "SELECT * FROM tbl WHERE col = 'abc'", + // https://github.com/XiaoMi/soar/issues/169 + "SELECT * FROM tbl WHERE col = 'abc' and c = 1", + "update tb set c = 1 where a = 2 and b = 3", + "delete from tb where a = 2 and b = 3", }, } diff --git a/common/config.go b/common/config.go index a8a78bb..c916818 100644 --- a/common/config.go +++ b/common/config.go @@ -512,7 +512,7 @@ func readCmdFlags() error { maxGroupByColsCount := flag.Int("max-group-by-cols-count", Config.MaxGroupByColsCount, "MaxGroupByColsCount, 单条 SQL 中 GroupBy 包含列的最大数量") maxDistinctCount := flag.Int("max-distinct-count", Config.MaxDistinctCount, "MaxDistinctCount, 单条 SQL 中 Distinct 的最大数量") maxIdxColsCount := flag.Int("max-index-cols-count", Config.MaxIdxColsCount, "MaxIdxColsCount, 复合索引中包含列的最大数量") - maxTextColsCount := flag.Int("max-texst-cols-count", Config.MaxTextColsCount, "MaxTextColsCount, 表中含有的 text/blob 列的最大数量") + maxTextColsCount := flag.Int("max-text-cols-count", Config.MaxTextColsCount, "MaxTextColsCount, 表中含有的 text/blob 列的最大数量") maxTotalRows := flag.Int64("max-total-rows", Config.MaxTotalRows, "MaxTotalRows, 计算散粒度时，当数据行数大于MaxTotalRows即开启数据库保护模式，不计算散粒度") maxQueryCost := flag.Int64("max-query-cost", Config.MaxQueryCost, "MaxQueryCost, last_query_cost 超过该值时将给予警告") spaghettiQueryLength := flag.Int("spaghetti-query-length", Config.SpaghettiQueryLength, "SpaghettiQueryLength, SQL最大长度警告，超过该长度会给警告") diff --git a/vendor/github.com/pingcap/parser/ast/expressions.go b/vendor/github.com/pingcap/parser/ast/expressions.go index dd0302a..1804040 100644 --- a/vendor/github.com/pingcap/parser/ast/expressions.go +++ b/vendor/github.com/pingcap/parser/ast/expressions.go @@ -154,7 +154,17 @@ type BinaryOperationExpr struct { // Restore implements Node interface. func (n *BinaryOperationExpr) Restore(ctx *RestoreCtx) error { - return errors.New("Not implemented") + if err := n.L.Restore(ctx); err != nil { + return errors.Annotate(err, "An error occurred when restore BinaryOperationExpr.L") + } + if err := n.Op.Restore(ctx.In); err != nil { + return errors.Annotate(err, "An error occurred when restore BinaryOperationExpr.Op") + } + if err := n.R.Restore(ctx); err != nil { + return errors.Annotate(err, "An error occurred when restore BinaryOperationExpr.R") + } + + return nil } // Format the ExprNode into a Writer. @@ -200,7 +210,15 @@ type WhenClause struct { // Restore implements Node interface. func (n *WhenClause) Restore(ctx *RestoreCtx) error { - return errors.New("Not implemented") + ctx.WriteKeyWord("WHEN ") + if err := n.Expr.Restore(ctx); err != nil { + return errors.Annotate(err, "An error occurred while restore WhenClauses.Expr") + } + ctx.WriteKeyWord(" THEN ") + if err := n.Result.Restore(ctx); err != nil { + return errors.Annotate(err, "An error occurred while restore WhenClauses.Result") + } + return nil } // Accept implements Node Accept interface. @@ -238,7 +256,28 @@ type CaseExpr struct { // Restore implements Node interface. func (n *CaseExpr) Restore(ctx *RestoreCtx) error { - return errors.New("Not implemented") + ctx.WriteKeyWord("CASE") + if n.Value != nil { + ctx.WritePlain(" ") + if err := n.Value.Restore(ctx); err != nil { + return errors.Annotate(err, "An error occurred while restore CaseExpr.Value") + } + } + for _, clause := range n.WhenClauses { + ctx.WritePlain(" ") + if err := clause.Restore(ctx); err != nil { + return errors.Annotate(err, "An error occurred while restore CaseExpr.WhenClauses") + } + } + if n.ElseClause != nil { + ctx.WriteKeyWord(" ELSE ") + if err := n.ElseClause.Restore(ctx); err != nil { + return errors.Annotate(err, "An error occurred while restore CaseExpr.ElseClause") + } + } + ctx.WriteKeyWord(" END") + + return nil } // Format the ExprNode into a Writer. @@ -777,7 +816,12 @@ type ParenthesesExpr struct { // Restore implements Node interface. func (n *ParenthesesExpr) Restore(ctx *RestoreCtx) error { - return errors.New("Not implemented") + ctx.WritePlain("(") + if err := n.Expr.Restore(ctx); err != nil { + return errors.Annotate(err, "An error occurred when restore ParenthesesExpr.Expr") + } + ctx.WritePlain(")") + return nil } // Format the ExprNode into a Writer. diff --git a/vendor/vendor.json b/vendor/vendor.json index f60b573..d1839ea 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -107,104 +107,104 @@ { "checksumSHA1": "fjZ4hf54cBBlV260wsAMZJ5T3po=", "path": "github.com/pingcap/parser", - "revision": "b7f4816358997259f649a06fd126db3542b644c8", - "revisionTime": "2018-12-17T08:17:10Z" + "revision": "deacf026787e7b6867b7083dcc461393ac700224", + "revisionTime": "2018-12-18T07:19:12Z" }, { - "checksumSHA1": "nRo2Z/Gyb1ziK+CKQi1OAOMa4hc=", + "checksumSHA1": "dk8IFRGPUrMidLa7QfmI/prTXTA=", "path": "github.com/pingcap/parser/ast", - "revision": "b7f4816358997259f649a06fd126db3542b644c8", - "revisionTime": "2018-12-17T08:17:10Z" + "revision": "deacf026787e7b6867b7083dcc461393ac700224", + "revisionTime": "2018-12-18T07:19:12Z" }, { "checksumSHA1": "skWGV4FNvD3vr+5olepaPPnylUw=", "path": "github.com/pingcap/parser/auth", - "revision": "b7f4816358997259f649a06fd126db3542b644c8", - "revisionTime": "2018-12-17T08:17:10Z" + "revision": "deacf026787e7b6867b7083dcc461393ac700224", + "revisionTime": "2018-12-18T07:19:12Z" }, { "checksumSHA1": "t4UHo966WzU9Z0IJkyGHRp0loOk=", "path": "github.com/pingcap/parser/charset", - "revision": "b7f4816358997259f649a06fd126db3542b644c8", - "revisionTime": "2018-12-17T08:17:10Z" + "revision": "deacf026787e7b6867b7083dcc461393ac700224", + "revisionTime": "2018-12-18T07:19:12Z" }, { "checksumSHA1": "SInoXbsRe0tnBwmatmtZYfSFbdk=", "path": "github.com/pingcap/parser/format", - "revision": "b7f4816358997259f649a06fd126db3542b644c8", - "revisionTime": "2018-12-17T08:17:10Z" + "revision": "deacf026787e7b6867b7083dcc461393ac700224", + "revisionTime": "2018-12-18T07:19:12Z" }, { "checksumSHA1": "WZYTGDMnc1UfTdjdZoBbISsnpRY=", "path": "github.com/pingcap/parser/model", - "revision": "b7f4816358997259f649a06fd126db3542b644c8", - "revisionTime": "2018-12-17T08:17:10Z" + "revision": "deacf026787e7b6867b7083dcc461393ac700224", + "revisionTime": "2018-12-18T07:19:12Z" }, { "checksumSHA1": "QBa9yiMDQNl2cLLwqlRoNTpCPNg=", "path": "github.com/pingcap/parser/mysql", - "revision": "b7f4816358997259f649a06fd126db3542b644c8", - "revisionTime": "2018-12-17T08:17:10Z" + "revision": "deacf026787e7b6867b7083dcc461393ac700224", + "revisionTime": "2018-12-18T07:19:12Z" }, { "checksumSHA1": "+O6CYIE0jT8pIDxWRP9FtKwFZjI=", "path": "github.com/pingcap/parser/opcode", - "revision": "b7f4816358997259f649a06fd126db3542b644c8", - "revisionTime": "2018-12-17T08:17:10Z" + "revision": "deacf026787e7b6867b7083dcc461393ac700224", + "revisionTime": "2018-12-18T07:19:12Z" }, { "checksumSHA1": "XvnUllvwMYd6HrMvMiKnn4cGN2M=", "path": "github.com/pingcap/parser/terror", - "revision": "b7f4816358997259f649a06fd126db3542b644c8", - "revisionTime": "2018-12-17T08:17:10Z" + "revision": "deacf026787e7b6867b7083dcc461393ac700224", + "revisionTime": "2018-12-18T07:19:12Z" }, { "checksumSHA1": "YoDiJ3sniNqxkP5X/BDkx6efteA=", "path": "github.com/pingcap/parser/types", - "revision": "b7f4816358997259f649a06fd126db3542b644c8", - "revisionTime": "2018-12-17T08:17:10Z" + "revision": "deacf026787e7b6867b7083dcc461393ac700224", + "revisionTime": "2018-12-18T07:19:12Z" }, { "checksumSHA1": "fWqL/7jTYOiqDNmiUcQi3u45Hw0=", "path": "github.com/pingcap/tidb/sessionctx/stmtctx", - "revision": "ab88f0b1c3b6db829f3e553963673a9bf7752899", - "revisionTime": "2018-12-17T12:08:05Z" + "revision": "4899ce5e923b5833cc684ea8d301f73773718330", + "revisionTime": "2018-12-18T06:47:23Z" }, { "checksumSHA1": "kXyszfR2fQ6bHvuCCFlHRkt1mF0=", "path": "github.com/pingcap/tidb/types", - "revision": "ab88f0b1c3b6db829f3e553963673a9bf7752899", - "revisionTime": "2018-12-17T12:08:05Z" + "revision": "4899ce5e923b5833cc684ea8d301f73773718330", + "revisionTime": "2018-12-18T06:47:23Z" }, { "checksumSHA1": "DWVD7+ygtT66IQ+cqXmMJ5OVqUk=", "path": "github.com/pingcap/tidb/types/json", - "revision": "ab88f0b1c3b6db829f3e553963673a9bf7752899", - "revisionTime": "2018-12-17T12:08:05Z" + "revision": "4899ce5e923b5833cc684ea8d301f73773718330", + "revisionTime": "2018-12-18T06:47:23Z" }, { "checksumSHA1": "6vi/eCZXqNTa5eAUpxDZet4LPlY=", "path": "github.com/pingcap/tidb/types/parser_driver", - "revision": "ab88f0b1c3b6db829f3e553963673a9bf7752899", - "revisionTime": "2018-12-17T12:08:05Z" + "revision": "4899ce5e923b5833cc684ea8d301f73773718330", + "revisionTime": "2018-12-18T06:47:23Z" }, { "checksumSHA1": "s709bhSrG2Ec35406mGtrySid4s=", "path": "github.com/pingcap/tidb/util/execdetails", - "revision": "ab88f0b1c3b6db829f3e553963673a9bf7752899", - "revisionTime": "2018-12-17T12:08:05Z" + "revision": "4899ce5e923b5833cc684ea8d301f73773718330", + "revisionTime": "2018-12-18T06:47:23Z" }, { "checksumSHA1": "nUC7zVoAMNR2a+z2iGqHoN2AkFE=", "path": "github.com/pingcap/tidb/util/hack", - "revision": "ab88f0b1c3b6db829f3e553963673a9bf7752899", - "revisionTime": "2018-12-17T12:08:05Z" + "revision": "4899ce5e923b5833cc684ea8d301f73773718330", + "revisionTime": "2018-12-18T06:47:23Z" }, { "checksumSHA1": "xSyepiuqsoaaeDch7cXeumvVHKM=", "path": "github.com/pingcap/tidb/util/memory", - "revision": "ab88f0b1c3b6db829f3e553963673a9bf7752899", - "revisionTime": "2018-12-17T12:08:05Z" + "revision": "4899ce5e923b5833cc684ea8d301f73773718330", + "revisionTime": "2018-12-18T06:47:23Z" }, { "checksumSHA1": "SmYeIK/fIYXNu8IKxD6HOVQVTuU=", @@ -401,62 +401,62 @@ { "checksumSHA1": "aKn1oKcY74N8TRLm3Ayt7Q4bbI4=", "path": "vitess.io/vitess/go/bytes2", - "revision": "935bf8ac1ad1e75bc8fd26a74fbfc906f15aa7d0", - "revisionTime": "2018-12-17T04:01:44Z" + "revision": "bbff624145ae9e6c797b2a1b9e04fdddc92af3a9", + "revisionTime": "2018-12-18T04:30:55Z" }, { "checksumSHA1": "JVCEN4UGRmg3TofIBdzZMZ3G0Ww=", "path": "vitess.io/vitess/go/hack", - "revision": "935bf8ac1ad1e75bc8fd26a74fbfc906f15aa7d0", - "revisionTime": "2018-12-17T04:01:44Z" + "revision": "bbff624145ae9e6c797b2a1b9e04fdddc92af3a9", + "revisionTime": "2018-12-18T04:30:55Z" }, { "checksumSHA1": "F5pcGq+2W1FHEjgktTdKOE6W8mk=", "path": "vitess.io/vitess/go/sqltypes", - "revision": "935bf8ac1ad1e75bc8fd26a74fbfc906f15aa7d0", - "revisionTime": "2018-12-17T04:01:44Z" + "revision": "bbff624145ae9e6c797b2a1b9e04fdddc92af3a9", + "revisionTime": "2018-12-18T04:30:55Z" }, { "checksumSHA1": "ntFIQYkBS51G6y+FEkjFW40+HOU=", "path": "vitess.io/vitess/go/vt/log", - "revision": "935bf8ac1ad1e75bc8fd26a74fbfc906f15aa7d0", - "revisionTime": "2018-12-17T04:01:44Z" + "revision": "bbff624145ae9e6c797b2a1b9e04fdddc92af3a9", + "revisionTime": "2018-12-18T04:30:55Z" }, { "checksumSHA1": "tPQFPwbMdjuX0qjNl4Zl8zc37JQ=", "path": "vitess.io/vitess/go/vt/proto/query", - "revision": "935bf8ac1ad1e75bc8fd26a74fbfc906f15aa7d0", - "revisionTime": "2018-12-17T04:01:44Z" + "revision": "bbff624145ae9e6c797b2a1b9e04fdddc92af3a9", + "revisionTime": "2018-12-18T04:30:55Z" }, { "checksumSHA1": "o0tR/c7lgr0pLkxk7CdvjiNDAKU=", "path": "vitess.io/vitess/go/vt/proto/topodata", - "revision": "935bf8ac1ad1e75bc8fd26a74fbfc906f15aa7d0", - "revisionTime": "2018-12-17T04:01:44Z" + "revision": "bbff624145ae9e6c797b2a1b9e04fdddc92af3a9", + "revisionTime": "2018-12-18T04:30:55Z" }, { "checksumSHA1": "77UojBqi0yyeQvR70j7C3kcKclQ=", "path": "vitess.io/vitess/go/vt/proto/vtgate", - "revision": "935bf8ac1ad1e75bc8fd26a74fbfc906f15aa7d0", - "revisionTime": "2018-12-17T04:01:44Z" + "revision": "bbff624145ae9e6c797b2a1b9e04fdddc92af3a9", + "revisionTime": "2018-12-18T04:30:55Z" }, { "checksumSHA1": "QpWGhoVDwM+8+sgYLI/YU+95iGU=", "path": "vitess.io/vitess/go/vt/proto/vtrpc", - "revision": "935bf8ac1ad1e75bc8fd26a74fbfc906f15aa7d0", - "revisionTime": "2018-12-17T04:01:44Z" + "revision": "bbff624145ae9e6c797b2a1b9e04fdddc92af3a9", + "revisionTime": "2018-12-18T04:30:55Z" }, { "checksumSHA1": "lENrUY/YyxwYFHYN+21TBH92P3U=", "path": "vitess.io/vitess/go/vt/sqlparser", - "revision": "935bf8ac1ad1e75bc8fd26a74fbfc906f15aa7d0", - "revisionTime": "2018-12-17T04:01:44Z" + "revision": "bbff624145ae9e6c797b2a1b9e04fdddc92af3a9", + "revisionTime": "2018-12-18T04:30:55Z" }, { "checksumSHA1": "Jx+gOh/kiBDSZxEIWHyYn9brjdo=", "path": "vitess.io/vitess/go/vt/vterrors", - "revision": "935bf8ac1ad1e75bc8fd26a74fbfc906f15aa7d0", - "revisionTime": "2018-12-17T04:01:44Z" + "revision": "bbff624145ae9e6c797b2a1b9e04fdddc92af3a9", + "revisionTime": "2018-12-18T04:30:55Z" } ], "rootPath": "github.com/XiaoMi/soar" -- GitLab