fix #99 add -check-config parameter

  -check-config will check
  * YAML format
  * OnlineDSN connection
  * TestDSN connection

  add doc for blacklist config

cmd/soar/soar.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/XiaoMi/soar/advisor"
@@ -49,27 +48,8 @@ func main() {
 	// 配置文件&命令行参数解析
 	initConfig()
 
-	// 打印支持启发式建议
-	if common.Config.ListHeuristicRules {
-		// 只打印支持的优化建议
-		advisor.ListHeuristicRules(advisor.HeuristicRules)
-		return
-	}
-	// 打印支持的 SQL 重写规则
-	if common.Config.ListRewriteRules {
-		ast.ListRewriteRules(ast.RewriteRules)
-		return
-	}
-	// 打印所有的测试 SQL
-	if common.Config.ListTestSqls {
-		advisor.ListTestSQLs()
-		return
-	}
-	// 打印支持的 report-type
-	if common.Config.ListReportTypes {
-		common.ListReportTypes()
-		return
-	}
+	// 命令行帮助工具，如 -list-report-types, -check-config等。
+	helpTools()
 
 	// 环境初始化，连接检查线上环境+构建测试环境
 	vEnv, rEnv := env.BuildEnv()
@@ -87,7 +67,7 @@ func main() {
 
 	// 当程序卡死的时候，或者由于某些原因程序没有退出，可以通过捕获信号量的形式让程序优雅退出并且清理测试环境
 	common.HandleSignal(func() {
-		shutdown(vEnv, rEnv)
+		shutdown(vEnv)
 	})
 
 	// 对指定的库表进行索引重复检查
@@ -502,37 +482,3 @@ func main() {
 		return
 	}
 }
-
-func initConfig() {
-	// 更新 binary 文件所在路径为 BaseDir
-	ex, err := os.Executable()
-	if err != nil {
-		panic(err)
-	}
-	common.BaseDir = filepath.Dir(ex)
-
-	for i, c := range os.Args {
-		// 如果指定了 -config, 它必须是第一个参数
-		if strings.HasPrefix(c, "-config") && i != 1 {
-			fmt.Println("-config must be the first arg")
-			os.Exit(1)
-		}
-		// 等号两边请不要加空格
-		if c == "=" {
-			// -config = soar.yaml not support
-			fmt.Println("wrong format, no space between '=', eg: -config=soar.yaml")
-			os.Exit(1)
-		}
-	}
-
-	// 加载配置文件，处理命令行参数
-	err = common.ParseConfig(common.ArgConfig())
-	common.LogIfWarn(err, "")
-}
-
-func shutdown(vEnv *env.VirtualEnv, rEnv *database.Connector) {
-	if common.Config.DropTestTemporary {
-		vEnv.CleanUp()
-	}
-	os.Exit(0)
-}

cmd/soar/tool.go

+/*
+ * Copyright 2018 Xiaomi, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/XiaoMi/soar/advisor"
+	"github.com/XiaoMi/soar/ast"
+	"github.com/XiaoMi/soar/common"
+	"github.com/XiaoMi/soar/database"
+	"github.com/XiaoMi/soar/env"
+)
+
+// initConfig load config from default->file->cmdFlag
+func initConfig() {
+	// 更新 binary 文件所在路径为 BaseDir
+	ex, err := os.Executable()
+	if err != nil {
+		panic(err)
+	}
+	common.BaseDir = filepath.Dir(ex)
+
+	for i, c := range os.Args {
+		// 如果指定了 -config, 它必须是第一个参数
+		if strings.HasPrefix(c, "-config") && i != 1 {
+			fmt.Println("-config must be the first arg")
+			os.Exit(1)
+		}
+		// 等号两边请不要加空格
+		if c == "=" {
+			// -config = soar.yaml not support
+			fmt.Println("wrong format, no space between '=', eg: -config=soar.yaml")
+			os.Exit(1)
+		}
+	}
+
+	// 加载配置文件，处理命令行参数
+	err = common.ParseConfig(common.ArgConfig())
+	// 检查配置文件及命令行参数是否正确
+	if common.CheckConfig && err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+	common.LogIfWarn(err, "")
+}
+
+// checkConfig for `-check-config` flag
+// if error found return non-zero, no error return zero
+func checkConfig() int {
+	// TestDSN connection check
+	testConn := &database.Connector{
+		Addr:     common.Config.TestDSN.Addr,
+		User:     common.Config.TestDSN.User,
+		Pass:     common.Config.TestDSN.Password,
+		Database: common.Config.TestDSN.Schema,
+		Charset:  common.Config.TestDSN.Charset,
+	}
+	testVersion, err := testConn.Version()
+	if err != nil && !common.Config.TestDSN.Disable {
+		fmt.Println("test-dsn:", testConn, err.Error())
+		return 1
+	}
+	if common.Config.Verbose {
+		if err == nil {
+			fmt.Println("test-dsn", testConn, "Version:", testVersion)
+		} else {
+			fmt.Println("test-dsn", common.Config.TestDSN)
+		}
+	}
+	// OnlineDSN connection check
+	onlineConn := &database.Connector{
+		Addr:     common.Config.OnlineDSN.Addr,
+		User:     common.Config.OnlineDSN.User,
+		Pass:     common.Config.OnlineDSN.Password,
+		Database: common.Config.OnlineDSN.Schema,
+		Charset:  common.Config.OnlineDSN.Charset,
+	}
+	onlineVersion, err := onlineConn.Version()
+	if err != nil && !common.Config.OnlineDSN.Disable {
+		fmt.Println("online-dsn:", onlineConn, err.Error())
+		return 1
+	}
+	if common.Config.Verbose {
+		if err == nil {
+			fmt.Println("online-dsn", onlineConn, "Version:", onlineVersion)
+		} else {
+			fmt.Println("online-dsn", common.Config.OnlineDSN)
+		}
+	}
+	return 0
+}
+
+// helpTools help tools in cmd flags
+func helpTools() {
+	// environment error check, eg. MySQL password error
+	if common.CheckConfig {
+		os.Exit(checkConfig())
+	}
+	// 打印 SOAR 版本信息
+	if common.PrintVersion {
+		common.SoarVersion()
+		os.Exit(0)
+	}
+	// 打印已加载配置的各配置项，检查配置是否生效
+	if common.PrintConfig {
+		common.PrintConfiguration()
+		os.Exit(0)
+	}
+	// 打印支持启发式建议
+	if common.Config.ListHeuristicRules {
+		advisor.ListHeuristicRules(advisor.HeuristicRules)
+		os.Exit(0)
+	}
+	// 打印支持的 SQL 重写规则
+	if common.Config.ListRewriteRules {
+		ast.ListRewriteRules(ast.RewriteRules)
+		os.Exit(0)
+	}
+	// 打印所有的测试 SQL
+	if common.Config.ListTestSqls {
+		advisor.ListTestSQLs()
+		os.Exit(0)
+	}
+	// 打印支持的 report-type
+	if common.Config.ListReportTypes {
+		common.ListReportTypes()
+		os.Exit(0)
+	}
+}
+
+func shutdown(vEnv *env.VirtualEnv) {
+	if common.Config.DropTestTemporary {
+		vEnv.CleanUp()
+	}
+	os.Exit(0)
+}

common/config.go

@@ -32,12 +32,21 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-// BlackList 黑名单中的SQL不会被评审
-var BlackList []string
-var hasParsed bool
+var (
+	// BlackList 黑名单中的SQL不会被评审
+	BlackList []string
+	// PrintConfig -print-config
+	PrintConfig bool
+	// PrintVersion -print-config
+	PrintVersion bool
+	// CheckConfig -check-config
+	CheckConfig bool
+	// 防止 readCmdFlags 函数重入
+	hasParsed bool
+)
 
-// Configration 配置文件定义结构体
-type Configration struct {
+// Configuration 配置文件定义结构体
+type Configuration struct {
 	// +++++++++++++++测试环境+++++++++++++++++
 	OnlineDSN               *dsn   `yaml:"online-dsn"`                // 线上环境数据库配置
 	TestDSN                 *dsn   `yaml:"test-dsn"`                  // 测试环境数据库配置
@@ -131,7 +140,7 @@ func getDefaultLogOutput() string {
 }
 
 // Config 默认设置
-var Config = &Configration{
+var Config = &Configuration{
 	OnlineDSN: &dsn{
 		Schema:  "information_schema",
 		Charset: "utf8mb4",
@@ -369,7 +378,8 @@ func FormatDSN(env *dsn) string {
 	return fmt.Sprintf("%s:%s@%s/%s?charset=%s", env.User, env.Password, env.Addr, env.Schema, env.Charset)
 }
 
-func version() {
+// SoarVersion soar version information
+func SoarVersion() {
 	fmt.Println("Version:", Version)
 	fmt.Println("Branch:", Branch)
 	fmt.Println("Compile:", Compile)
@@ -444,8 +454,19 @@ func usage() {
 	}
 }
 
+// PrintConfiguration for `-print-config` flag
+func PrintConfiguration() {
+	// 打印配置的时候密码不显示
+	if !Config.Verbose {
+		Config.OnlineDSN.Password = "********"
+		Config.TestDSN.Password = "********"
+	}
+	data, _ := yaml.Marshal(Config)
+	fmt.Print(string(data))
+}
+
 // 加载配置文件
-func (conf *Configration) readConfigFile(path string) error {
+func (conf *Configuration) readConfigFile(path string) error {
 	configFile, err := os.Open(path)
 	if err != nil {
 		Log.Warning("readConfigFile(%s) os.Open failed: %v", path, err)
@@ -539,7 +560,8 @@ func readCmdFlags() error {
 	showLastQueryCost := flag.Bool("show-last-query-cost", Config.ShowLastQueryCost, "ShowLastQueryCost")
 	// +++++++++++++++++其他+++++++++++++++++++
 	printConfig := flag.Bool("print-config", false, "Print configs")
-	ver := flag.Bool("version", false, "Print version info")
+	checkConfig := flag.Bool("check-config", false, "Check configs")
+	printVersion := flag.Bool("version", false, "Print version info")
 	query := flag.String("query", Config.Query, "待评审的 SQL 或 SQL 文件，如 SQL 中包含特殊字符建议使用文件名。")
 	listHeuristicRules := flag.Bool("list-heuristic-rules", Config.ListHeuristicRules, "ListHeuristicRules, 打印支持的评审规则列表")
 	listRewriteRules := flag.Bool("list-rewrite-rules", Config.ListRewriteRules, "ListRewriteRules, 打印支持的重写规则列表")
@@ -647,21 +669,9 @@ func readCmdFlags() error {
 	Config.MaxPrettySQLLength = *maxPrettySQLLength
 	Config.MaxVarcharLength = *maxVarcharLength
 
-	if *ver {
-		version()
-		os.Exit(0)
-	}
-
-	if *printConfig {
-		// 打印配置的时候密码不显示
-		if !Config.Verbose {
-			Config.OnlineDSN.Password = "********"
-			Config.TestDSN.Password = "********"
-		}
-		data, _ := yaml.Marshal(Config)
-		fmt.Print(string(data))
-		os.Exit(0)
-	}
+	PrintVersion = *printVersion
+	PrintConfig = *printConfig
+	CheckConfig = *checkConfig
 
 	hasParsed = true
 	return nil
@@ -697,6 +707,7 @@ func ParseConfig(configFile string) error {
 	err = readCmdFlags()
 	if err != nil {
 		Log.Error("ParseConfig readCmdFlags Error: %v", err)
+		return err
 	}
 
 	// parse blacklist & ignore blacklist file parse error

common/config_test.go

@@ -47,7 +47,7 @@ func TestParseConfig(t *testing.T) {
 
 func TestReadConfigFile(t *testing.T) {
 	if Config == nil {
-		Config = new(Configration)
+		Config = new(Configuration)
 	}
 	Config.readConfigFile("../soar.yaml")
 }
@@ -110,3 +110,9 @@ func TestArgConfig(t *testing.T) {
 		}
 	}
 }
+
+func TestPrintConfiguration(t *testing.T) {
+	Config.Verbose = true
+	PrintConfiguration()
+
+}

common/logger.go

@@ -39,13 +39,14 @@ func init() {
 // LoggerInit Log配置初始化
 func LoggerInit() {
 	Log.SetLevel(Config.LogLevel)
-	if Config.LogOutput == "console" {
-		err := Log.SetLogger("console")
+	if Config.LogOutput == logs.AdapterConsole {
+		err := Log.SetLogger(logs.AdapterConsole)
 		if err != nil {
 			fmt.Println(err.Error())
 		}
 	} else {
-		err := Log.SetLogger("file", fmt.Sprintf(`{"filename":"%s","level":7,"maxlines":0,"maxsize":0,"daily":false,"maxdays":0}`, Config.LogOutput))
+		func() { _ = Log.DelLogger(logs.AdapterFile) }()
+		err := Log.SetLogger(logs.AdapterFile, fmt.Sprintf(`{"filename":"%s","level":7,"maxlines":0,"maxsize":0,"daily":false,"maxdays":0}`, Config.LogOutput))
 		if err != nil {
 			fmt.Println(err.Error())
 		}

common/testdata/TestPrintConfiguration.golden

+online-dsn:
+  addr: ""
+  schema: information_schema
+  user: ""
+  password: ""
+  charset: utf8mb4
+  disable: true
+test-dsn:
+  addr: ""
+  schema: information_schema
+  user: ""
+  password: ""
+  charset: utf8mb4
+  disable: true
+allow-online-as-test: false
+drop-test-temporary: true
+cleanup-test-database: false
+only-syntax-check: false
+sampling-statistic-target: 100
+sampling: false
+profiling: false
+trace: false
+explain: true
+conn-time-out: 3
+query-time-out: 30
+delimiter: ;
+log-level: 3
+log-output: /dev/stderr
+report-type: markdown
+report-css: ""
+report-javascript: ""
+report-title: SQL优化分析报告
+markdown-extensions: 94
+markdown-html-flags: 0
+ignore-rules:
+- COL.011
+rewrite-rules:
+- delimiter
+- orderbynull
+- groupbyconst
+- dmlorderby
+- having
+- star2columns
+- insertcolumns
+- distinctstar
+blacklist: ""
+max-join-table-count: 5
+max-group-by-cols-count: 5
+max-distinct-count: 5
+max-index-cols-count: 5
+max-total-rows: 9999999
+max-query-cost: 9999
+spaghetti-query-length: 2048
+allow-drop-index: false
+max-in-count: 10
+max-index-bytes-percolumn: 767
+max-index-bytes: 3072
+table-allow-charsets:
+- utf8
+- utf8mb4
+table-allow-engines:
+- innodb
+max-index-count: 10
+max-column-count: 40
+index-prefix: idx_
+unique-key-prefix: uk_
+max-subquery-depth: 5
+max-varchar-length: 1024
+explain-sql-report-type: pretty
+explain-type: extended
+explain-format: traditional
+explain-warn-select-type:
+- ""
+explain-warn-access-type:
+- ALL
+explain-max-keys: 3
+explain-min-keys: 0
+explain-max-rows: 10000
+explain-warn-extra:
+- Using temporary
+- Using filesort
+explain-max-filtered: 100
+explain-warn-scalability:
+- O(n)
+show-warnings: false
+show-last-query-cost: false
+query: ""
+list-heuristic-rules: false
+list-rewrite-rules: false
+list-test-sqls: false
+list-report-types: false
+verbose: true
+dry-run: true
+max-pretty-sql-length: 1024

database/mysql.go

@@ -62,7 +62,7 @@ func (db *Connector) NewConnection() mysql.Conn {
 func (db *Connector) Query(sql string, params ...interface{}) (*QueryResult, error) {
 	// 测试环境如果检查是关闭的，则SQL不会被执行
 	if common.Config.TestDSN.Disable {
-		return nil, errors.New("TestDsn Disable")
+		return nil, errors.New("Dsn Disable")
 	}
 
 	// 数据库安全性检查：如果 Connector 的 IP 端口与 TEST 环境不一致，则启用SQL白名单

database/profiling.go

@@ -51,7 +51,7 @@ func (db *Connector) Profiling(sql string, params ...interface{}) (*QueryResult,
 
 	// 测试环境如果检查是关闭的，则SQL不会被执行
 	if common.Config.TestDSN.Disable {
-		return nil, errors.New("TestDsn Disable")
+		return nil, errors.New("Dsn Disable")
 	}
 
 	// 数据库安全性检查：如果 Connector 的 IP 端口与 TEST 环境不一致，则启用 SQL 白名单

database/trace.go

@@ -60,7 +60,7 @@ func (db *Connector) Trace(sql string, params ...interface{}) (*QueryResult, err
 
 	// 测试环境如果检查是关闭的，则SQL不会被执行
 	if common.Config.TestDSN.Disable {
-		return nil, errors.New("TestDsn Disable")
+		return nil, errors.New("Dsn Disable")
 	}
 
 	// 数据库安全性检查：如果 Connector 的 IP 端口与 TEST 环境不一致，则启用SQL白名单

doc/config.md

@@ -36,7 +36,8 @@ log-output: ${your_log_dir}/soar.log
 report-type: markdown
 ignore-rules:
 - ""
-blacklist: ${BASE_DIR}/soar.blacklist
+# 黑名单中的 SQL 将不会给评审意见。一行一条 SQL，可以是正则也可以是指纹，填写指纹时注意问号需要加反斜线转义。
+blacklist: ${your_config_dir}/soar.blacklist
 # 启发式算法相关配置
 max-join-table-count: 5
 max-group-by-cols-count: 5