use DoltDB as built-in memory test-dsn

  without mysql test environment, use in memory mysql(DoltDB) as default
  test-dsn, soar can give index advice without mysql instance.

  in memory mysql(DoltDB) will listen a random port bind with localhost,
  and distroyed automatically when soar quit.

advisor/rules.go

@@ -1007,8 +1007,8 @@ func InitHeuristicRules() {
 			Severity: "L2",
 			Summary:  "建表语句中定义为 ON UPDATE CURRENT_TIMESTAMP 的字段不建议包含业务逻辑",
 			Content:  "定义为 ON UPDATE CURRENT_TIMESTAMP 的字段在该表其他字段更新时会联动修改，如果包含业务逻辑用户可见会埋下隐患。后续如有批量修改数据却又不想修改该字段时会导致数据错误。",
-			Case: `CREATE TABLE category (category_id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT,	name VARCHAR(25) NOT NULL, last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY  (category_id)`,
-			Func: (*Query4Audit).RuleCreateOnUpdate,
+			Case:     `CREATE TABLE category (category_id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT,	name VARCHAR(25) NOT NULL, last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY  (category_id)`,
+			Func:     (*Query4Audit).RuleCreateOnUpdate,
 		},
 		"RES.011": {
 			Item:     "RES.011",

cmd/soar/tool.go

@@ -41,8 +41,9 @@ func initConfig() {
 
 	for i, c := range os.Args {
 		// 如果指定了 -config, 它必须是第一个参数
-		if strings.HasPrefix(c, "-config") && i != 1 {
-			fmt.Println("-config must be the first arg")
+		if (strings.HasPrefix(c, "-config") || strings.HasPrefix(c, "--config")) &&
+			i != 1 {
+			fmt.Println("--config must be the first arg")
 			os.Exit(1)
 		}
 		// 等号两边请不要加空格

common/config.go

@@ -976,8 +976,8 @@ func ListReportTypes() {
 // ArgConfig get -config arg value from cli
 func ArgConfig() string {
 	var configFile string
-	if len(os.Args) > 1 && strings.HasPrefix(os.Args[1], "-config") {
-		if os.Args[1] == "-config" && len(os.Args) > 2 {
+	if len(os.Args) > 1 && (strings.HasPrefix(os.Args[1], "-config") || strings.HasPrefix(os.Args[1], "--config")) {
+		if (os.Args[1] == "-config" || os.Args[1] == "--config") && len(os.Args) > 2 {
 			if os.Args[2] == "=" && len(os.Args) > 3 {
 				// -config = soar.yaml not support
 				fmt.Println("wrong format, no space between '=', eg: -config=soar.yaml")
@@ -996,7 +996,7 @@ func ArgConfig() string {
 		}
 	} else {
 		for i, c := range os.Args {
-			if strings.HasPrefix(c, "-config") && i != 1 {
+			if (strings.HasPrefix(c, "-config") || strings.HasPrefix(c, "--config")) && i != 1 {
 				fmt.Println("-config must be the first arg")
 			}
 		}

common/config_test.go

@@ -133,8 +133,11 @@ func TestArgConfig(t *testing.T) {
 	}
 	testArgs2 := [][]string{
 		{"soar", "-config", "soar.yaml"},
+		{"soar", "--config", "soar.yaml"},
 		{"soar", "-config", "=soar.yaml"},
+		{"soar", "--config", "=soar.yaml"},
 		{"soar", "-config=soar.yaml"},
+		{"soar", "--config=soar.yaml"},
 	}
 	for _, args := range testArgs1 {
 		os.Args = args

env/dolt.go

+/*
+ * Copyright 2018 Xiaomi, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package env
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/XiaoMi/soar/common"
+
+	sqle "github.com/dolthub/go-mysql-server"
+	"github.com/dolthub/go-mysql-server/memory"
+	"github.com/dolthub/go-mysql-server/server"
+	"github.com/dolthub/go-mysql-server/sql/information_schema"
+	"github.com/sirupsen/logrus"
+)
+
+func NewDoltDB() {
+	// go-mysql-server set log
+	logrus.SetLevel(logrus.FatalLevel)
+
+	// random unused port
+	// https://www.lifewire.com/port-0-in-tcp-and-udp-818145
+	listener, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		common.Log.Warn(err.Error())
+		return
+	}
+	port := listener.Addr().(*net.TCPAddr).Port
+	if err = listener.Close(); err != nil {
+		common.Log.Error(err.Error())
+		return
+	}
+
+	// new in memory database server
+	var db string
+	switch common.Config.TestDSN.Schema {
+	case "", "information_schema", "mysql":
+		db = "test"
+	default:
+		db = common.Config.TestDSN.Schema
+	}
+	engine := sqle.NewDefault(
+		memory.NewMemoryDBProvider(
+			memory.NewDatabase(db),
+			information_schema.NewInformationSchemaDatabase(),
+		),
+	)
+	pass := fmt.Sprint(time.Now().Nanosecond())
+	engine.Analyzer.Catalog.MySQLDb.AddSuperUser("soar", "localhost", pass)
+	config := server.Config{
+		Protocol: "tcp",
+		Address:  fmt.Sprintf("localhost:%d", port),
+	}
+	s, err := server.NewDefaultServer(config, engine)
+	if err != nil {
+		common.Log.Error(err.Error())
+		return
+	}
+	common.Config.TestDSN.User = "soar"
+	common.Config.TestDSN.Password = pass
+	common.Config.TestDSN.Net = "tcp"
+	common.Config.TestDSN.Addr = fmt.Sprintf("localhost:%d", port)
+
+	common.Log.Info("starting built-in dolt database, address: localhost:%d", port)
+	err = s.Start()
+	common.LogIfError(err, "")
+}

env/env.go

@@ -44,23 +44,43 @@ type VirtualEnv struct {
 }
 
 // NewVirtualEnv 初始化一个新的测试环境
-func NewVirtualEnv(vEnv *database.Connector) *VirtualEnv {
-	return &VirtualEnv{
-		Connector: vEnv,
+func NewVirtualEnv() *VirtualEnv {
+	conn, err := database.NewConnector(common.Config.TestDSN)
+	common.LogIfError(err, "")
+
+	// validate test-dsn
+	_, err = conn.Version()
+	if err != nil && common.Config.TestDSN.Disable != true {
+		common.Log.Warn("test-dsn: %s, try to start built-in dolt database", err.Error())
+		go NewDoltDB()
+		for i := 0; i < 3; i++ {
+			time.Sleep(5 * time.Millisecond)
+			conn, err = database.NewConnector(common.Config.TestDSN)
+			common.LogIfError(err, "")
+			err = conn.Conn.Ping()
+			if err == nil {
+				common.Log.Info("built-in dolt database started, address: %s", common.Config.TestDSN.Addr)
+				break
+			}
+		}
+	}
+
+	vEnv := &VirtualEnv{
+		Connector: conn,
 		DBRef:     make(map[string]string),
 		Hash2DB:   make(map[string]string),
 		TableMap:  make(map[string]map[string]string),
 	}
+
+	return vEnv
 }
 
 // BuildEnv 测试环境初始化&连接线上环境检查
 // @output *VirtualEnv	测试环境
 // @output *database.Connector 线上环境连接句柄
 func BuildEnv() (*VirtualEnv, *database.Connector) {
-	connTest, err := database.NewConnector(common.Config.TestDSN)
-	common.LogIfError(err, "")
 	// 生成测试环境
-	vEnv := NewVirtualEnv(connTest)
+	vEnv := NewVirtualEnv()
 
 	// 检查测试环境可用性，并记录数据库版本
 	vEnvVersion, err := vEnv.Version()
@@ -420,24 +440,29 @@ func (vEnv *VirtualEnv) createDatabase(rEnv *database.Connector) error {
 }
 
 /*
-	@input:
-		database.Connector 为一个线上环境数据库连接句柄的复制，因为在处理SQL时需要对上下文进行关联处理，
-		所以存在修改DB连接参数（主要是数据库名称变更）的可能性，为了不影响整体上下文的环境，所以需要一个镜像句柄来做当前环境的操作。
-
-		dbName, tbName: 需要在环境中操作的库表名称，
-
-	@output:
-		return 执行过程中的错误
-
-	NOTE:
-		该函数会将线上环境中使用到的库表结构复制到测试环境中，为后续操作提供基础环境。
-		传入的库表名称均来自于对AST的解析，库表名称的获取遵循以下原则：
-			如果未在SQL中指定数据库名称，则数据库一定是配置文件（或命令行参数传入DSN）中指定的数据库
-			如果一个SQL中存在多个数据库，则只能有一个数据库是没有在SQL中被显示指定的（即DSN中指定的数据库）
-	TODO:
-		在一些可能的情况下，由于数据库配置的不一致（如SQL_MODE不同）导致remote环境的库表无法正确的在测试环境进行同步，
-		soar 能够做出判断并进行 session 级别的修改，但是这一阶段可用性保证应该是由用户提供两个完全相同（或测试环境兼容线上环境）
-		的数据库环境来实现的。
+@input:
+
+	database.Connector 为一个线上环境数据库连接句柄的复制，因为在处理SQL时需要对上下文进行关联处理，
+	所以存在修改DB连接参数（主要是数据库名称变更）的可能性，为了不影响整体上下文的环境，所以需要一个镜像句柄来做当前环境的操作。
+
+	dbName, tbName: 需要在环境中操作的库表名称，
+
+@output:
+
+	return 执行过程中的错误
+
+NOTE:
+
+	该函数会将线上环境中使用到的库表结构复制到测试环境中，为后续操作提供基础环境。
+	传入的库表名称均来自于对AST的解析，库表名称的获取遵循以下原则：
+		如果未在SQL中指定数据库名称，则数据库一定是配置文件（或命令行参数传入DSN）中指定的数据库
+		如果一个SQL中存在多个数据库，则只能有一个数据库是没有在SQL中被显示指定的（即DSN中指定的数据库）
+
+TODO:
+
+	在一些可能的情况下，由于数据库配置的不一致（如SQL_MODE不同）导致remote环境的库表无法正确的在测试环境进行同步，
+	soar 能够做出判断并进行 session 级别的修改，但是这一阶段可用性保证应该是由用户提供两个完全相同（或测试环境兼容线上环境）
+	的数据库环境来实现的。
 */
 func (vEnv *VirtualEnv) createTable(rEnv *database.Connector, tbName string) error {
 	// 判断数据库是否已经创建

go.mod

 module github.com/XiaoMi/soar
 
-go 1.15
+go 1.19
 
 require (
-	github.com/Azure/azure-storage-blob-go v0.10.0 // indirect
-	github.com/Azure/go-autorest/autorest v0.10.0 // indirect
 	github.com/CorgiMan/json2 v0.0.0-20150213135156-e72957aba209
-	github.com/aquarapid/vaultlib v0.5.1 // indirect
 	github.com/astaxie/beego v1.12.3
-	github.com/buger/jsonparser v1.1.1 // indirect
-	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
-	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 // indirect
 	github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5
+	github.com/dolthub/go-mysql-server v0.12.1-0.20221019203727-b99ff94e4329
 	github.com/gedex/inflector v0.0.0-20170307190818-16278e9db813
-	github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab // indirect
 	github.com/go-sql-driver/mysql v1.6.0
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/gorilla/handlers v1.5.1 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/hashicorp/consul/api v1.5.0 // indirect
-	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
-	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
-	github.com/hashicorp/go-uuid v1.0.2 // indirect
-	github.com/hashicorp/serf v0.9.2 // indirect
-	github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c // indirect
-	github.com/jeremywohl/flatten v0.0.0-20190921043622-d936035e55cf // indirect
-	github.com/klauspost/pgzip v1.2.4 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
 	github.com/kr/pretty v0.2.1
-	github.com/looplab/fsm v0.2.0 // indirect
-	github.com/martini-contrib/auth v0.0.0-20150219114609-fa62c19b7ae8 // indirect
-	github.com/martini-contrib/gzip v0.0.0-20151124214156-6c035326b43f // indirect
-	github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11 // indirect
-	github.com/mitchellh/go-ps v1.0.0 // indirect
-	github.com/mitchellh/go-testing-interface v1.14.0 // indirect
-	github.com/mitchellh/mapstructure v1.2.3 // indirect
-	github.com/montanaflynn/stats v0.6.3 // indirect
-	github.com/olekukonko/tablewriter v0.0.5-0.20200416053754-163badb3bac6 // indirect
-	github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/percona/go-mysql v0.0.0-20210427141028-73d29c6da78c
-	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 // indirect
 	github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307
-	github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 // indirect
 	github.com/pingcap/tidb v1.1.0-beta.0.20210601085537-5d7c852770eb
-	github.com/pingcap/tipb v0.0.0-20210601083426-79a378b6d1c4 // indirect
-	github.com/planetscale/pargzip v0.0.0-20201116224723-90c7fc03ea8a // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
 	github.com/russross/blackfriday v1.6.0
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca
-	github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e // indirect
+	github.com/sirupsen/logrus v1.9.0
+	github.com/tidwall/gjson v1.12.1
+	gopkg.in/yaml.v2 v2.4.0
+	vitess.io/vitess v0.0.0-20200325000816-eda961851d63
+)
+
+require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
+	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
+	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
+	github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dolthub/vitess v0.0.0-20221004165409-08281765376f // indirect
+	github.com/go-kit/kit v0.10.0 // indirect
+	github.com/go-ole/go-ole v1.2.4 // indirect
+	github.com/gocraft/dbr/v2 v2.7.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/flatbuffers v2.0.6+incompatible // indirect
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lestrrat-go/strftime v1.0.4 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mitchellh/hashstructure v1.1.0 // indirect
+	github.com/montanaflynn/stats v0.6.3 // indirect
+	github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 // indirect
+	github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd // indirect
+	github.com/pingcap/kvproto v0.0.0-20210507054410-a8152f8a876c // indirect
+	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 // indirect
+	github.com/pingcap/tipb v0.0.0-20210601083426-79a378b6d1c4 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_golang v1.7.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.10.0 // indirect
+	github.com/prometheus/procfs v0.1.3 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
+	github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect
 	github.com/shirou/gopsutil v3.21.2+incompatible // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/sjmudd/stopwatch v0.0.0-20170613150411-f380bf8a9be1 // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/cobra v1.1.1 // indirect
-	github.com/spyzhov/ajson v0.4.2 // indirect
-	github.com/tidwall/gjson v1.12.1
+	github.com/stretchr/testify v1.7.1 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d // indirect
+	github.com/uber/jaeger-client-go v2.22.1+incompatible // indirect
+	github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
+	go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b // indirect
+	go.opentelemetry.io/otel v1.7.0 // indirect
+	go.opentelemetry.io/otel/trace v1.7.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.17.0 // indirect
-	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
-	golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b // indirect
-	golang.org/x/text v0.3.6 // indirect
-	gopkg.in/gcfg.v1 v1.2.3 // indirect
-	gopkg.in/warnings.v0 v0.1.2 // indirect
-	gopkg.in/yaml.v2 v2.4.0
-	vitess.io/vitess v0.0.0-20200325000816-eda961851d63
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.10 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/genproto v0.0.0-20210506142907-4a47615972c2 // indirect
+	google.golang.org/grpc v1.37.0 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
+
+replace google.golang.org/grpc => google.golang.org/grpc v1.26.0