From 5f5e5e69e64c1836d6766727b6e473f1ec07fe5f Mon Sep 17 00:00:00 2001 From: Leon Zhang Date: Tue, 13 Nov 2018 21:27:25 +0800 Subject: [PATCH] fix #99 add -check-config parameter -check-config will check * YAML format * OnlineDSN connection * TestDSN connection add doc for blacklist config --- cmd/soar/soar.go | 60 +------ cmd/soar/tool.go | 154 ++++++++++++++++++ common/config.go | 59 ++++--- common/config_test.go | 8 +- common/logger.go | 7 +- common/testdata/TestPrintConfiguration.golden | 94 +++++++++++ database/mysql.go | 2 +- database/profiling.go | 2 +- database/trace.go | 2 +- doc/config.md | 3 +- 10 files changed, 302 insertions(+), 89 deletions(-) create mode 100644 cmd/soar/tool.go create mode 100644 common/testdata/TestPrintConfiguration.golden diff --git a/cmd/soar/soar.go b/cmd/soar/soar.go index d23fe6a..3a38d91 100644 --- a/cmd/soar/soar.go +++ b/cmd/soar/soar.go @@ -21,7 +21,6 @@ import ( "fmt" "io/ioutil" "os" - "path/filepath" "strings" "github.com/XiaoMi/soar/advisor" @@ -49,27 +48,8 @@ func main() { // 配置文件&命令行参数解析 initConfig() - // 打印支持启发式建议 - if common.Config.ListHeuristicRules { - // 只打印支持的优化建议 - advisor.ListHeuristicRules(advisor.HeuristicRules) - return - } - // 打印支持的 SQL 重写规则 - if common.Config.ListRewriteRules { - ast.ListRewriteRules(ast.RewriteRules) - return - } - // 打印所有的测试 SQL - if common.Config.ListTestSqls { - advisor.ListTestSQLs() - return - } - // 打印支持的 report-type - if common.Config.ListReportTypes { - common.ListReportTypes() - return - } + // 命令行帮助工具，如 -list-report-types, -check-config等。 + helpTools() // 环境初始化，连接检查线上环境+构建测试环境 vEnv, rEnv := env.BuildEnv() @@ -87,7 +67,7 @@ func main() { // 当程序卡死的时候，或者由于某些原因程序没有退出，可以通过捕获信号量的形式让程序优雅退出并且清理测试环境 common.HandleSignal(func() { - shutdown(vEnv, rEnv) + shutdown(vEnv) }) // 对指定的库表进行索引重复检查 @@ -502,37 +482,3 @@ func main() { return } } - -func initConfig() { - // 更新 binary 文件所在路径为 BaseDir - ex, err := os.Executable() - if err != nil { - panic(err) - } - common.BaseDir = filepath.Dir(ex) - - for i, c := range os.Args { - // 如果指定了 -config, 它必须是第一个参数 - if strings.HasPrefix(c, "-config") && i != 1 { - fmt.Println("-config must be the first arg") - os.Exit(1) - } - // 等号两边请不要加空格 - if c == "=" { - // -config = soar.yaml not support - fmt.Println("wrong format, no space between '=', eg: -config=soar.yaml") - os.Exit(1) - } - } - - // 加载配置文件，处理命令行参数 - err = common.ParseConfig(common.ArgConfig()) - common.LogIfWarn(err, "") -} - -func shutdown(vEnv *env.VirtualEnv, rEnv *database.Connector) { - if common.Config.DropTestTemporary { - vEnv.CleanUp() - } - os.Exit(0) -} diff --git a/cmd/soar/tool.go b/cmd/soar/tool.go new file mode 100644 index 0000000..96f0f55 --- /dev/null +++ b/cmd/soar/tool.go @@ -0,0 +1,154 @@ +/* + * Copyright 2018 Xiaomi, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/XiaoMi/soar/advisor" + "github.com/XiaoMi/soar/ast" + "github.com/XiaoMi/soar/common" + "github.com/XiaoMi/soar/database" + "github.com/XiaoMi/soar/env" +) + +// initConfig load config from default->file->cmdFlag +func initConfig() { + // 更新 binary 文件所在路径为 BaseDir + ex, err := os.Executable() + if err != nil { + panic(err) + } + common.BaseDir = filepath.Dir(ex) + + for i, c := range os.Args { + // 如果指定了 -config, 它必须是第一个参数 + if strings.HasPrefix(c, "-config") && i != 1 { + fmt.Println("-config must be the first arg") + os.Exit(1) + } + // 等号两边请不要加空格 + if c == "=" { + // -config = soar.yaml not support + fmt.Println("wrong format, no space between '=', eg: -config=soar.yaml") + os.Exit(1) + } + } + + // 加载配置文件，处理命令行参数 + err = common.ParseConfig(common.ArgConfig()) + // 检查配置文件及命令行参数是否正确 + if common.CheckConfig && err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + common.LogIfWarn(err, "") +} + +// checkConfig for `-check-config` flag +// if error found return non-zero, no error return zero +func checkConfig() int { + // TestDSN connection check + testConn := &database.Connector{ + Addr: common.Config.TestDSN.Addr, + User: common.Config.TestDSN.User, + Pass: common.Config.TestDSN.Password, + Database: common.Config.TestDSN.Schema, + Charset: common.Config.TestDSN.Charset, + } + testVersion, err := testConn.Version() + if err != nil && !common.Config.TestDSN.Disable { + fmt.Println("test-dsn:", testConn, err.Error()) + return 1 + } + if common.Config.Verbose { + if err == nil { + fmt.Println("test-dsn", testConn, "Version:", testVersion) + } else { + fmt.Println("test-dsn", common.Config.TestDSN) + } + } + // OnlineDSN connection check + onlineConn := &database.Connector{ + Addr: common.Config.OnlineDSN.Addr, + User: common.Config.OnlineDSN.User, + Pass: common.Config.OnlineDSN.Password, + Database: common.Config.OnlineDSN.Schema, + Charset: common.Config.OnlineDSN.Charset, + } + onlineVersion, err := onlineConn.Version() + if err != nil && !common.Config.OnlineDSN.Disable { + fmt.Println("online-dsn:", onlineConn, err.Error()) + return 1 + } + if common.Config.Verbose { + if err == nil { + fmt.Println("online-dsn", onlineConn, "Version:", onlineVersion) + } else { + fmt.Println("online-dsn", common.Config.OnlineDSN) + } + } + return 0 +} + +// helpTools help tools in cmd flags +func helpTools() { + // environment error check, eg. MySQL password error + if common.CheckConfig { + os.Exit(checkConfig()) + } + // 打印 SOAR 版本信息 + if common.PrintVersion { + common.SoarVersion() + os.Exit(0) + } + // 打印已加载配置的各配置项，检查配置是否生效 + if common.PrintConfig { + common.PrintConfiguration() + os.Exit(0) + } + // 打印支持启发式建议 + if common.Config.ListHeuristicRules { + advisor.ListHeuristicRules(advisor.HeuristicRules) + os.Exit(0) + } + // 打印支持的 SQL 重写规则 + if common.Config.ListRewriteRules { + ast.ListRewriteRules(ast.RewriteRules) + os.Exit(0) + } + // 打印所有的测试 SQL + if common.Config.ListTestSqls { + advisor.ListTestSQLs() + os.Exit(0) + } + // 打印支持的 report-type + if common.Config.ListReportTypes { + common.ListReportTypes() + os.Exit(0) + } +} + +func shutdown(vEnv *env.VirtualEnv) { + if common.Config.DropTestTemporary { + vEnv.CleanUp() + } + os.Exit(0) +} diff --git a/common/config.go b/common/config.go index da93b31..8f5a0d8 100644 --- a/common/config.go +++ b/common/config.go @@ -32,12 +32,21 @@ import ( "gopkg.in/yaml.v2" ) -// BlackList 黑名单中的SQL不会被评审 -var BlackList []string -var hasParsed bool +var ( + // BlackList 黑名单中的SQL不会被评审 + BlackList []string + // PrintConfig -print-config + PrintConfig bool + // PrintVersion -print-config + PrintVersion bool + // CheckConfig -check-config + CheckConfig bool + // 防止 readCmdFlags 函数重入 + hasParsed bool +) -// Configration 配置文件定义结构体 -type Configration struct { +// Configuration 配置文件定义结构体 +type Configuration struct { // +++++++++++++++测试环境+++++++++++++++++ OnlineDSN *dsn `yaml:"online-dsn"` // 线上环境数据库配置 TestDSN *dsn `yaml:"test-dsn"` // 测试环境数据库配置 @@ -131,7 +140,7 @@ func getDefaultLogOutput() string { } // Config 默认设置 -var Config = &Configration{ +var Config = &Configuration{ OnlineDSN: &dsn{ Schema: "information_schema", Charset: "utf8mb4", @@ -369,7 +378,8 @@ func FormatDSN(env *dsn) string { return fmt.Sprintf("%s:%s@%s/%s?charset=%s", env.User, env.Password, env.Addr, env.Schema, env.Charset) } -func version() { +// SoarVersion soar version information +func SoarVersion() { fmt.Println("Version:", Version) fmt.Println("Branch:", Branch) fmt.Println("Compile:", Compile) @@ -444,8 +454,19 @@ func usage() { } } +// PrintConfiguration for `-print-config` flag +func PrintConfiguration() { + // 打印配置的时候密码不显示 + if !Config.Verbose { + Config.OnlineDSN.Password = "********" + Config.TestDSN.Password = "********" + } + data, _ := yaml.Marshal(Config) + fmt.Print(string(data)) +} + // 加载配置文件 -func (conf *Configration) readConfigFile(path string) error { +func (conf *Configuration) readConfigFile(path string) error { configFile, err := os.Open(path) if err != nil { Log.Warning("readConfigFile(%s) os.Open failed: %v", path, err) @@ -539,7 +560,8 @@ func readCmdFlags() error { showLastQueryCost := flag.Bool("show-last-query-cost", Config.ShowLastQueryCost, "ShowLastQueryCost") // +++++++++++++++++其他+++++++++++++++++++ printConfig := flag.Bool("print-config", false, "Print configs") - ver := flag.Bool("version", false, "Print version info") + checkConfig := flag.Bool("check-config", false, "Check configs") + printVersion := flag.Bool("version", false, "Print version info") query := flag.String("query", Config.Query, "待评审的 SQL 或 SQL 文件，如 SQL 中包含特殊字符建议使用文件名。") listHeuristicRules := flag.Bool("list-heuristic-rules", Config.ListHeuristicRules, "ListHeuristicRules, 打印支持的评审规则列表") listRewriteRules := flag.Bool("list-rewrite-rules", Config.ListRewriteRules, "ListRewriteRules, 打印支持的重写规则列表") @@ -647,21 +669,9 @@ func readCmdFlags() error { Config.MaxPrettySQLLength = *maxPrettySQLLength Config.MaxVarcharLength = *maxVarcharLength - if *ver { - version() - os.Exit(0) - } - - if *printConfig { - // 打印配置的时候密码不显示 - if !Config.Verbose { - Config.OnlineDSN.Password = "********" - Config.TestDSN.Password = "********" - } - data, _ := yaml.Marshal(Config) - fmt.Print(string(data)) - os.Exit(0) - } + PrintVersion = *printVersion + PrintConfig = *printConfig + CheckConfig = *checkConfig hasParsed = true return nil @@ -697,6 +707,7 @@ func ParseConfig(configFile string) error { err = readCmdFlags() if err != nil { Log.Error("ParseConfig readCmdFlags Error: %v", err) + return err } // parse blacklist & ignore blacklist file parse error diff --git a/common/config_test.go b/common/config_test.go index 8dcf6c5..7f9a569 100644 --- a/common/config_test.go +++ b/common/config_test.go @@ -47,7 +47,7 @@ func TestParseConfig(t *testing.T) { func TestReadConfigFile(t *testing.T) { if Config == nil { - Config = new(Configration) + Config = new(Configuration) } Config.readConfigFile("../soar.yaml") } @@ -110,3 +110,9 @@ func TestArgConfig(t *testing.T) { } } } + +func TestPrintConfiguration(t *testing.T) { + Config.Verbose = true + PrintConfiguration() + +} diff --git a/common/logger.go b/common/logger.go index bbde98b..77059cb 100644 --- a/common/logger.go +++ b/common/logger.go @@ -39,13 +39,14 @@ func init() { // LoggerInit Log配置初始化 func LoggerInit() { Log.SetLevel(Config.LogLevel) - if Config.LogOutput == "console" { - err := Log.SetLogger("console") + if Config.LogOutput == logs.AdapterConsole { + err := Log.SetLogger(logs.AdapterConsole) if err != nil { fmt.Println(err.Error()) } } else { - err := Log.SetLogger("file", fmt.Sprintf(`{"filename":"%s","level":7,"maxlines":0,"maxsize":0,"daily":false,"maxdays":0}`, Config.LogOutput)) + func() { _ = Log.DelLogger(logs.AdapterFile) }() + err := Log.SetLogger(logs.AdapterFile, fmt.Sprintf(`{"filename":"%s","level":7,"maxlines":0,"maxsize":0,"daily":false,"maxdays":0}`, Config.LogOutput)) if err != nil { fmt.Println(err.Error()) } diff --git a/common/testdata/TestPrintConfiguration.golden b/common/testdata/TestPrintConfiguration.golden new file mode 100644 index 0000000..09bd65a --- /dev/null +++ b/common/testdata/TestPrintConfiguration.golden @@ -0,0 +1,94 @@ +online-dsn: + addr: "" + schema: information_schema + user: "" + password: "" + charset: utf8mb4 + disable: true +test-dsn: + addr: "" + schema: information_schema + user: "" + password: "" + charset: utf8mb4 + disable: true +allow-online-as-test: false +drop-test-temporary: true +cleanup-test-database: false +only-syntax-check: false +sampling-statistic-target: 100 +sampling: false +profiling: false +trace: false +explain: true +conn-time-out: 3 +query-time-out: 30 +delimiter: ; +log-level: 3 +log-output: /dev/stderr +report-type: markdown +report-css: "" +report-javascript: "" +report-title: SQL优化分析报告 +markdown-extensions: 94 +markdown-html-flags: 0 +ignore-rules: +- COL.011 +rewrite-rules: +- delimiter +- orderbynull +- groupbyconst +- dmlorderby +- having +- star2columns +- insertcolumns +- distinctstar +blacklist: "" +max-join-table-count: 5 +max-group-by-cols-count: 5 +max-distinct-count: 5 +max-index-cols-count: 5 +max-total-rows: 9999999 +max-query-cost: 9999 +spaghetti-query-length: 2048 +allow-drop-index: false +max-in-count: 10 +max-index-bytes-percolumn: 767 +max-index-bytes: 3072 +table-allow-charsets: +- utf8 +- utf8mb4 +table-allow-engines: +- innodb +max-index-count: 10 +max-column-count: 40 +index-prefix: idx_ +unique-key-prefix: uk_ +max-subquery-depth: 5 +max-varchar-length: 1024 +explain-sql-report-type: pretty +explain-type: extended +explain-format: traditional +explain-warn-select-type: +- "" +explain-warn-access-type: +- ALL +explain-max-keys: 3 +explain-min-keys: 0 +explain-max-rows: 10000 +explain-warn-extra: +- Using temporary +- Using filesort +explain-max-filtered: 100 +explain-warn-scalability: +- O(n) +show-warnings: false +show-last-query-cost: false +query: "" +list-heuristic-rules: false +list-rewrite-rules: false +list-test-sqls: false +list-report-types: false +verbose: true +dry-run: true +max-pretty-sql-length: 1024 diff --git a/database/mysql.go b/database/mysql.go index 3409e44..170d6f0 100644 --- a/database/mysql.go +++ b/database/mysql.go @@ -62,7 +62,7 @@ func (db *Connector) NewConnection() mysql.Conn { func (db *Connector) Query(sql string, params ...interface{}) (*QueryResult, error) { // 测试环境如果检查是关闭的，则SQL不会被执行 if common.Config.TestDSN.Disable { - return nil, errors.New("TestDsn Disable") + return nil, errors.New("Dsn Disable") } // 数据库安全性检查：如果 Connector 的 IP 端口与 TEST 环境不一致，则启用SQL白名单 diff --git a/database/profiling.go b/database/profiling.go index 638126c..a607231 100644 --- a/database/profiling.go +++ b/database/profiling.go @@ -51,7 +51,7 @@ func (db *Connector) Profiling(sql string, params ...interface{}) (*QueryResult, // 测试环境如果检查是关闭的，则SQL不会被执行 if common.Config.TestDSN.Disable { - return nil, errors.New("TestDsn Disable") + return nil, errors.New("Dsn Disable") } // 数据库安全性检查：如果 Connector 的 IP 端口与 TEST 环境不一致，则启用 SQL 白名单 diff --git a/database/trace.go b/database/trace.go index 7938a91..0fe890a 100644 --- a/database/trace.go +++ b/database/trace.go @@ -60,7 +60,7 @@ func (db *Connector) Trace(sql string, params ...interface{}) (*QueryResult, err // 测试环境如果检查是关闭的，则SQL不会被执行 if common.Config.TestDSN.Disable { - return nil, errors.New("TestDsn Disable") + return nil, errors.New("Dsn Disable") } // 数据库安全性检查：如果 Connector 的 IP 端口与 TEST 环境不一致，则启用SQL白名单 diff --git a/doc/config.md b/doc/config.md index a76fbab..50ef2b1 100644 --- a/doc/config.md +++ b/doc/config.md @@ -36,7 +36,8 @@ log-output: ${your_log_dir}/soar.log report-type: markdown ignore-rules: - "" -blacklist: ${BASE_DIR}/soar.blacklist +# 黑名单中的 SQL 将不会给评审意见。一行一条 SQL，可以是正则也可以是指纹，填写指纹时注意问号需要加反斜线转义。 +blacklist: ${your_config_dir}/soar.blacklist # 启发式算法相关配置 max-join-table-count: 5 max-group-by-cols-count: 5 -- GitLab