Merge branch 'master' into go-sql-driver

	修改：     doc/example/main_test.sh

CHANGES.md

 # CHANGELOG
 
-## 2018-11
+## 2018-12
+
 - DOING: english translation
+
+## 2018-11
+
+- add all third-party lib into vendor
+- support `-report-type chardet`
+- add more heuristic rules: TBL.008, KEY.010, ARG.012, KWR.004
 - add -cleanup-test-database command-line arg
+- add -check-config parameter
+- fix #146 pretty cause syntax error
+- fix #140 COL.012, COL.015 NULL type about TEXT/BLOB
+- fix #141 empty output when query execute failed on mysql
+- fix #89 index advisor give wrong database name, `optimizer_xx`
+- fix #121 RemoveSQLComment trim space
+- fix #120 trimspace before check single line comment
+- fix mac os stdout print buffer truncate
 - fix -config arg load file error
-- fix #87 RuleImplicitConversion value type mistach check bug
+- fix #116 SplitStatement check if single comment line is in multi-line sql.
+- fix #112 multi-line comment will cause line counter error, when -report-type=lint
+- fix #110 remove bom before auditing
+- fix #104 case insensitive regex @ CLA.009
+- fix #87 RuleImplicitConversion value type mismatch check bug
 - fix #38 always true where condition check
 - abandon stdin terminal interactive mod, which may seems like hangup
 
 ## 2018-10
-- Fix SplitStatement mulitstatement eof bug #66
+
+- Fix SplitStatement multistatement eof bug #66
 - Fix pretty func hangup issue #47
 - Fix some foolish code spell error
 - Use travis for CI
-- Fix Go 1.8 defapth GOPATH compatible issue BUG #5
+- Fix Go 1.8 default GOPATH compatible issue BUG #5
 - 2018-10-20 开源先锋日(OSCAR)对外正式开源发布代码
 
 ## 2018-09
+
 - 修复多个启发式建议不准确BUG，优化部分建议文案使得建议更清晰
-- 基于TiDB Parser完善多个DDL类型语句的建议
+- 基于 TiDB Parser 完善多个 DDL 类型语句的建议
 - 新增lint report-type类型，支持Vim Plugin优化建议输出
 - 更新整理项目文档，开源准备
 - 2018-09-21 Gdevops SOAR首次对外进行技术分享宣传
 
 ## 2018-08
-- 利用docker临时容器进行daily测试
+
+- 利用 docker 临时容器进行 daily 测试
 - 添加main_test全功能回归测试
 - 修复在测试中发现的问题
-- mymysql合并MySQL8.0相关PR，修改vendor依赖
+- mymysql 合并 MySQL8.0 相关PR，修改vendor依赖
 - 改善HeuristicRule中的文案
 - 持续集成Vitess Parser的改进
-- NewQuery4Audit结构体中引入TiDB Parser
+- NewQuery4Audit 结构体中引入 TiDB Parser
 - 通过TiAST完成大量与 DDL 相关的TODO
 - 修改heuristic rules检查的返回值，提升拓展性
 - 建议中引入Position，用于表示建议产生于SQL的位置
@@ -41,22 +63,25 @@
 - 优化 doc 文档
 
 ## 2018-07
+
 - 补充文档，添加项目LOGO
 - 改善代码质量提升测试覆盖度
 - mymysql升级，支持MySQL 8.0
 - 提供remove-comment小工具
 - 提供索引重复检查小工具
-- HeuristicRule新增RuleSpaceAfterDot
+- HeuristicRule 新增 RuleSpaceAfterDot
 - 支持字符集和Collation不相同时的隐式数据类型转换的检查
 
 ## 2018-06
+
 - 支持更多的SQL Rewrite规则
 - 添加SQL执行超时限制
 - 索引优化建议支持对约束的检查
-- 修复数据采样中null值处理不正确的问题
-- Explain支持last_query_cost
+- 修复数据采样中 NULL 值处理不正确的问题
+- Explain 支持 last_query_cost
 
 ## 2018-05
+
 - 添加数据采样功能
 - 添加语句执行安全检查
 - 支持DDL语法检查
@@ -67,6 +92,7 @@
 - 支持SQL Pretty输出
 
 ## 2018-04
+
 - 支持语法检查
 - 支持测试环境
 - 支持MySQL原数据的获取
@@ -76,7 +102,8 @@
 - 引入配置文件
 
 ## 2018-03
+
 - 基本架构设计
 - 添加大量底层函数用于处理AST
-- 添加Insert、Delete、Update转写成Select的基本函数
+- 添加Insert、Delete、Update 转写成 Select 的基本函数
 - 支持MySQL Explain信息输出

Makefile

@@ -18,6 +18,7 @@ BUILD_TIME=`date +%Y%m%d%H%M`
 COMMIT_VERSION=`git rev-parse HEAD`
 
 # Add mysql version for testing `MYSQL_RELEASE=percona MYSQL_VERSION=5.7 make docker`
+# MySQL 5.1 `MYSQL_RELEASE=vsamov/mysql-5.1.73 make docker`
 # MYSQL_RELEASE: mysql, percona, mariadb ...
 # MYSQL_VERSION: latest, 8.0, 5.7, 5.6, 5.5 ...
 # use mysql:latest as default
@@ -45,8 +46,9 @@ deps:
 	@echo "\033[92mDependency check\033[0m"
 	@bash ./deps.sh
 	# The retool tools.json is setup from retool-install.sh
+	# some packages download need more open internet access
 	retool sync
-	retool do gometalinter.v2 intall
+	#retool do gometalinter.v2 --install
 
 # Code format
 .PHONY: fmt
@@ -64,6 +66,12 @@ test:
 	go test -race ./...
 	@echo "test Success!"
 
+# Rule golang test cases with `-update` flag
+test-update:
+	@echo "\033[92mRun all test cases with -update flag ...\033[0m"
+	go test ./... -update
+	@echo "test-update Success!"
+
 # Code Coverage
 # colorful coverage numerical >=90% GREEN, <80% RED, Other YELLOW
 .PHONY: cover
@@ -107,7 +115,7 @@ doc: build
 
 # Add or change a heuristic rule
 .PHONY: heuristic
-heuristic: doc docker
+heuristic: doc
 	@echo "\033[92mUpdate Heuristic rule golden files ...\033[0m"
 	go test github.com/XiaoMi/soar/advisor -v -update -run TestListHeuristicRules
 	go test github.com/XiaoMi/soar/advisor -v -update -run TestMergeConflictHeuristicRules
@@ -146,7 +154,7 @@ lint: build
 	@echo "gometalinter check your code is pretty good"
 
 .PHONY: release
-release: deps build
+release: build
 	@echo "\033[92mCross platform building for release ...\033[0m"
 	@mkdir -p release
 	@for GOOS in darwin linux windows; do \
@@ -163,6 +171,7 @@ release: deps build
 docker:
 	@echo "\033[92mBuild mysql test enviorment\033[0m"
 	@docker stop soar-mysql 2>/dev/null || true
+	@docker wait soar-mysql 2>/dev/null || true
 	@echo "docker run --name soar-mysql $(MYSQL_RELEASE):$(MYSQL_VERSION)"
 	@docker run --name soar-mysql --rm -d \
 	-e MYSQL_ROOT_PASSWORD=1tIsB1g3rt \
@@ -171,17 +180,22 @@ docker:
 	-v `pwd`/doc/example/sakila.sql.gz:/docker-entrypoint-initdb.d/sakila.sql.gz \
 	$(MYSQL_RELEASE):$(MYSQL_VERSION)
 
-	@echo -n "waiting for sakila database initializing "
-	@while ! mysql -h 127.0.0.1 -u root sakila -p1tIsB1g3rt -NBe "do 1;" 2>/dev/null; do \
+	@echo "waiting for sakila database initializing "
+	@while ! docker exec soar-mysql mysql --user=root --password=1tIsB1g3rt --host "127.0.0.1" --silent -NBe "do 1" >/dev/null 2>&1 ; do \
 	printf '.' ; \
 	sleep 1 ; \
 	done ; \
 	echo '.'
 	@echo "mysql test enviorment is ready!"
 
-.PHONY: connect
-connect:
-	mysql -h 127.0.0.1 -u root -p1tIsB1g3rt sakila -c
+.PHONY: docker-connect
+docker-connect:
+	docker exec -it soar-mysql mysql --user=root --password=1tIsB1g3rt --host "127.0.0.1"
+
+# attach docker container with bash interactive mode
+.PHONY: docker-it
+docker-it:
+	docker exec -it soar-mysql /bin/bash
 
 .PHONY: main_test
 main_test: install
@@ -196,7 +210,7 @@ daily: | deps fmt vendor docker cover doc lint release install main_test clean l
 
 # vendor, docker will cost long time, if all those are ready, daily-quick will much more fast.
 .PHONY: daily-quick
-daily-quick: | deps fmt cover doc lint logo
+daily-quick: | deps fmt cover main_test doc lint logo
 	@echo "\033[92mdaily-quick build finished\033[0m"
 
 .PHONY: logo

advisor/explainer_test.go

@@ -23,7 +23,7 @@ import (
 )
 
 func TestDigestExplainText(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	var text = `+----+-------------+---------+-------+---------------------------------------------------------+-------------------+---------+---------------------------+------+-------------+
 | id | select_type | table   | type  | possible_keys                                           | key               | key_len | ref                       | rows | Extra       |
 +----+-------------+---------+-------+---------------------------------------------------------+-------------------+---------+---------------------------+------+-------------+

advisor/heuristic_test.go

@@ -23,7 +23,9 @@ import (
 
 	"github.com/XiaoMi/soar/common"
 
+	"github.com/XiaoMi/soar/env"
 	"github.com/kr/pretty"
+	"vitess.io/vitess/go/vt/sqlparser"
 )
 
 // ALI.001
@@ -471,6 +473,8 @@ func TestRuleAddDefaultValue(t *testing.T) {
 			`ALTER TABLE test modify id varchar(10) DEFAULT '';`,
 			`ALTER TABLE test CHANGE id id varchar(10) DEFAULT '';`,
 			"create table test(id int not null default 0 comment '用户id')",
+			`create table tb (a text)`,
+			`alter table tb add a text`,
 		},
 	}
 	for _, sql := range sqls[0] {
@@ -936,6 +940,50 @@ func TestRuleLoadFile(t *testing.T) {
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
 
+// RES.009
+func TestRuleMultiCompare(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	sqls := [][]string{
+		{
+			"SELECT * FROM tbl WHERE col = col = 'abc'",
+			"UPDATE tbl set col = 1 WHERE col = col = 'abc'",
+			"DELETE FROM tbl WHERE col = col = 'abc'",
+		},
+		{
+			"SELECT * FROM tbl WHERE col = 'abc'",
+			// https://github.com/XiaoMi/soar/issues/169
+			"SELECT * FROM tbl WHERE col = 'abc' and c = 1",
+			"update tb set c = 1 where a = 2 and b = 3",
+			"delete from tb where a = 2 and b = 3",
+		},
+	}
+
+	for _, sql := range sqls[0] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleMultiCompare()
+			if rule.Item != "RES.009" {
+				t.Error("Rule not match:", rule.Item, "Expect : RES.009, SQL: ", sql)
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+
+	for _, sql := range sqls[1] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleMultiCompare()
+			if rule.Item != "OK" {
+				t.Error("Rule not match:", rule.Item, "Expect : OK, SQL: ", sql)
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}
+
 // STA.001
 func TestRuleStandardINEQ(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
@@ -2413,16 +2461,21 @@ func TestRuleAlterDropKey(t *testing.T) {
 // COL.012
 func TestRuleCantBeNull(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
-	sqls := []string{
-		"CREATE TABLE `tbl` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `c` longblob, PRIMARY KEY (`id`));",
-		"alter TABLE `tbl` add column `c` longblob;",
-		"alter TABLE `tbl` add column `c` text;",
-		"alter TABLE `tbl` add column `c` blob;",
+	sqls := [][]string{
+		{
+			"CREATE TABLE `tb`(`c` longblob NOT NULL);",
+		},
+		{
+			"CREATE TABLE `tbl` (`c` longblob);",
+			"alter TABLE `tbl` add column `c` longblob;",
+			"alter TABLE `tbl` add column `c` text;",
+			"alter TABLE `tbl` add column `c` blob;",
+		},
 	}
-	for _, sql := range sqls {
+	for _, sql := range sqls[0] {
 		q, err := NewQuery4Audit(sql)
 		if err == nil {
-			rule := q.RuleCantBeNull()
+			rule := q.RuleBLOBNotNull()
 			if rule.Item != "COL.012" {
 				t.Error("Rule not match:", rule.Item, "Expect : COL.012")
 			}
@@ -2430,6 +2483,18 @@ func TestRuleCantBeNull(t *testing.T) {
 			t.Error("sqlparser.Parse Error:", err)
 		}
 	}
+
+	for _, sql := range sqls[1] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleBLOBNotNull()
+			if rule.Item != "OK" {
+				t.Error("Rule not match:", rule.Item, "Expect : OK")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
 
@@ -2762,6 +2827,7 @@ func TestRuleTableCharsetCheck(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	sqls := [][]string{
 		{
+			"CREATE DATABASE sbtest /*!40100 DEFAULT CHARACTER SET latin1 */;",
 			"create table tbl (a int) DEFAULT CHARSET=latin1;",
 			"ALTER TABLE tbl CONVERT TO CHARACTER SET latin1;",
 		},
@@ -2796,6 +2862,44 @@ func TestRuleTableCharsetCheck(t *testing.T) {
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
 
+// TBL.008
+func TestRuleTableCollateCheck(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	sqls := [][]string{
+		{
+			"CREATE DATABASE sbtest /*!40100 DEFAULT COLLATE latin1_bin */;",
+			"create table tbl (a int) DEFAULT COLLATE=latin1_bin;",
+		},
+		{
+			"create table tlb (a int);",
+			"ALTER TABLE `tbl` add column a int, add column b int ;",
+		},
+	}
+	for _, sql := range sqls[0] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleTableCollateCheck()
+			if rule.Item != "TBL.008" {
+				t.Error("Rule not match:", rule.Item, "Expect : TBL.008")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+	for _, sql := range sqls[1] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleTableCollateCheck()
+			if rule.Item != "OK" {
+				t.Error("Rule not match:", rule.Item, "Expect : OK")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}
+
 // COL.015
 func TestRuleBlobDefaultValue(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
@@ -2806,7 +2910,13 @@ func TestRuleBlobDefaultValue(t *testing.T) {
 		},
 		{
 			"CREATE TABLE `tb` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `c` blob NOT NULL, PRIMARY KEY (`id`));",
-			"alter table `tb` add column `c` blob NOT NULL DEFAULT NULL;",
+			"CREATE TABLE `tb` (`col` text NOT NULL);",
+			"alter table `tb` add column `c` blob NOT NULL;",
+			"ALTER TABLE tb ADD COLUMN a BLOB DEFAULT NULL",
+			"CREATE TABLE tb ( a BLOB DEFAULT NULL)",
+			"alter TABLE `tbl` add column `c` longblob;",
+			"alter TABLE `tbl` add column `c` text;",
+			"alter TABLE `tbl` add column `c` blob;",
 		},
 	}
 
@@ -2924,6 +3034,89 @@ func TestRuleVarcharLength(t *testing.T) {
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
 
+// COL.018
+func TestRuleColumnNotAllowType(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+
+	sqls := [][]string{
+		{
+			"CREATE TABLE tab (a BOOLEAN);",
+			"CREATE TABLE tab (a BOOLEAN );",
+			"ALTER TABLE `tb` add column `a` BOOLEAN;",
+		},
+		{
+			"CREATE TABLE `tb` ( `id` varchar(1024));",
+			"ALTER TABLE `tb` add column `id` varchar(35);",
+		},
+	}
+
+	for _, sql := range sqls[0] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleColumnNotAllowType()
+			if rule.Item != "COL.018" {
+				t.Error("Rule not match:", rule.Item, "Expect : COL.018")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+
+	for _, sql := range sqls[1] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleColumnNotAllowType()
+			if rule.Item != "OK" {
+				t.Error("Rule not match:", rule.Item, "Expect : OK")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}
+
+// COL.019
+func TestRuleTimePrecision(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	sqls := [][]string{
+		// 正面的例子
+		{
+			"CREATE TABLE t1 (t TIME(3), dt DATETIME(6));",
+			"ALTER TABLE t1 add t TIME(3);",
+		},
+		// 反面的例子
+		{
+			"CREATE TABLE t1 (t TIME, dt DATETIME);",
+			"ALTER TABLE t1 add t TIME;",
+		},
+	}
+	for _, sql := range sqls[0] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleTimePrecision()
+			if rule.Item != "COL.019" {
+				t.Error("Rule not match:", rule.Item, "Expect : COL.019")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+
+	for _, sql := range sqls[1] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleTimePrecision()
+			if rule.Item != "OK" {
+				t.Error("Rule not match:", rule.Item, "Expect : OK")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}
+
 // KEY.002
 func TestRuleNoOSCKey(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
@@ -2985,6 +3178,100 @@ func TestRuleTooManyFields(t *testing.T) {
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
 
+// COL.007
+func TestRuleMaxTextColsCount(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	sqls := []string{
+		"create table tbl (a int, b text, c blob, d text);",
+	}
+
+	common.Config.MaxColCount = 0
+	for _, sql := range sqls {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleMaxTextColsCount()
+			if rule.Item != "COL.007" {
+				t.Error("Rule not match:", rule.Item, "Expect : COL.007")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}
+
+// COL.007
+func TestRuleMaxTextColsCountWithEnv(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	orgMaxTextColsCount := common.Config.MaxTextColsCount
+	common.Config.MaxTextColsCount = 1
+
+	vEnv, rEnv := env.BuildEnv()
+	defer vEnv.CleanUp()
+	initSQLs := []string{
+		`CREATE TABLE t1 (id int, title text);`,
+		`CREATE TABLE t2 (id int, title text);`,
+	}
+
+	for _, sql := range initSQLs {
+		vEnv.BuildVirtualEnv(rEnv, sql)
+	}
+
+	sqls := [][]string{
+		{
+			"alter table t1 add column other text;",
+		},
+		{
+			"alter table t2 add column col varchar(10);",
+		},
+	}
+
+	for _, sql := range sqls[0] {
+		vEnv.BuildVirtualEnv(rEnv, sql)
+		stmt, syntaxErr := sqlparser.Parse(sql)
+		if syntaxErr != nil {
+			t.Error(syntaxErr)
+		}
+
+		q := &Query4Audit{Query: sql, Stmt: stmt}
+		idxAdvisor, err := NewAdvisor(vEnv, *rEnv, *q)
+		if err != nil {
+			t.Error("NewAdvisor Error: ", err, "SQL: ", sql)
+		}
+
+		if idxAdvisor != nil {
+			rule := idxAdvisor.RuleMaxTextColsCount()
+			if rule.Item != "COL.007" {
+				t.Error("Rule not match:", rule, "Expect : COL.007, SQL:", sql)
+			}
+		}
+	}
+
+	for _, sql := range sqls[1] {
+		vEnv.BuildVirtualEnv(rEnv, sql)
+		stmt, syntaxErr := sqlparser.Parse(sql)
+		if syntaxErr != nil {
+			t.Error(syntaxErr)
+		}
+
+		q := &Query4Audit{Query: sql, Stmt: stmt}
+		idxAdvisor, err := NewAdvisor(vEnv, *rEnv, *q)
+		if err != nil {
+			t.Error("NewAdvisor Error: ", err, "SQL: ", sql)
+		}
+
+		if idxAdvisor != nil {
+			rule := idxAdvisor.RuleMaxTextColsCount()
+			if rule.Item != "OK" {
+				t.Error("Rule not match:", rule, "Expect : OK, SQL:", sql)
+			}
+		}
+	}
+
+	common.Config.MaxTextColsCount = orgMaxTextColsCount
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}
+
 // TBL.002
 func TestRuleAllowEngine(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
@@ -3194,7 +3481,7 @@ func TestRuleSpaceAfterDot(t *testing.T) {
 
 func TestRuleMySQLError(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
-	err := errors.New(`Received #1146 error from MySQL server: "can't xxxx"`)
+	err := errors.New(`received #1146 error from MySQL server: "can't xxxx"`)
 	if RuleMySQLError("ERR.002", err).Content != "" {
 		t.Error("Want: '', Bug get: ", err)
 	}

advisor/index.go

@@ -111,7 +111,11 @@ func NewAdvisor(env *env.VirtualEnv, rEnv database.Connector, q Query4Audit) (*I
 			}
 		}
 
-		return nil, nil
+		return &IndexAdvisor{
+			vEnv: env,
+			rEnv: rEnv,
+			Ast:  q.Stmt,
+		}, nil
 
 	case *sqlparser.DBDDL:
 		// 忽略建库语句
@@ -624,7 +628,6 @@ func (idxAdv *IndexAdvisor) buildJoinIndex(meta common.Meta) []IndexInfo {
 		indexColsList := make(map[string]map[string][]*common.Column)
 		for _, col := range IndexCols {
 			mergeIndex(indexColsList, col)
-
 		}
 
 		if common.Config.TestDSN.Disable || common.Config.OnlineDSN.Disable {
@@ -719,6 +722,11 @@ func (idxAdv *IndexAdvisor) buildIndexWithNoEnv(indexList map[string]map[string]
 
 // mergeIndex 将索引用到的列去重后合并到一起
 func mergeIndex(idxList map[string]map[string][]*common.Column, column *common.Column) {
+	// 散粒度低于阈值将不会添加索引
+	if common.Config.MinCardinality/100 > column.Cardinality {
+		return
+	}
+
 	db := column.DB
 	tb := column.Table
 	if idxList[db] == nil {
@@ -1010,11 +1018,12 @@ func (idxAdv *IndexAdvisor) HeuristicCheck(q Query4Audit) map[string]Rule {
 	}
 
 	ruleFuncs := []func(*IndexAdvisor) Rule{
+		(*IndexAdvisor).RuleMaxTextColsCount,   // COL.007
 		(*IndexAdvisor).RuleImplicitConversion, // ARG.003
+		(*IndexAdvisor).RuleGroupByConst,       // CLA.004
+		(*IndexAdvisor).RuleOrderByConst,       // CLA.005
+		(*IndexAdvisor).RuleUpdatePrimaryKey,   // CLA.016
 		// (*IndexAdvisor).RuleImpossibleOuterJoin, // TODO: JOI.003, JOI.004
-		(*IndexAdvisor).RuleGroupByConst,     // CLA.004
-		(*IndexAdvisor).RuleOrderByConst,     // CLA.005
-		(*IndexAdvisor).RuleUpdatePrimaryKey, // CLA.016
 	}
 
 	for _, f := range ruleFuncs {

advisor/index_test.go

@@ -35,7 +35,8 @@ var update = flag.Bool("update", false, "update .golden files")
 var vEnv *env.VirtualEnv
 var rEnv *database.Connector
 
-func init() {
+func TestMain(m *testing.M) {
+	// 初始化 init
 	common.BaseDir = common.DevPath
 	err := common.ParseConfig("")
 	common.LogIfError(err, "init ParseConfig")
@@ -50,7 +51,13 @@ func init() {
 		fmt.Println(err.Error(), ", By pass all advisor test cases")
 		os.Exit(0)
 	}
-	defer vEnv.CleanUp()
+
+	// 分割线
+	flag.Parse()
+	m.Run()
+
+	// 环境清理
+	vEnv.CleanUp()
 }
 
 // ARG.003
@@ -63,19 +70,27 @@ func TestRuleImplicitConversion(t *testing.T) {
 		`CREATE TABLE t1 (id int, title varchar(255) CHARSET utf8 COLLATE utf8_general_ci);`,
 		`CREATE TABLE t2 (id int, title varchar(255) CHARSET utf8mb4);`,
 		`CREATE TABLE t3 (id int, title varchar(255) CHARSET utf8 COLLATE utf8_bin);`,
+		`CREATE TABLE t4 (id int, col bit(1));`,
 	}
 	for _, sql := range initSQLs {
 		vEnv.BuildVirtualEnv(rEnv, sql)
 	}
 
-	sqls := []string{
-		"SELECT * FROM t1 WHERE title >= 60;",
-		"SELECT * FROM t1, t2 WHERE t1.title = t2.title;",
-		"SELECT * FROM t1, t3 WHERE t1.title = t3.title;",
-		"SELECT * FROM t1 WHERE title in (60, '60');",
-		"SELECT * FROM t1 WHERE title in (60);",
+	sqls := [][]string{
+		{
+			"SELECT * FROM t1 WHERE title >= 60;",
+			"SELECT * FROM t1, t2 WHERE t1.title = t2.title;",
+			"SELECT * FROM t1, t3 WHERE t1.title = t3.title;",
+			"SELECT * FROM t1 WHERE title in (60, '60');",
+			"SELECT * FROM t1 WHERE title in (60);",
+			"SELECT * FROM t4 WHERE col = '1'",
+		},
+		{
+			// https://github.com/XiaoMi/soar/issues/151
+			"SELECT * FROM t4 WHERE col = 1",
+		},
 	}
-	for _, sql := range sqls {
+	for _, sql := range sqls[0] {
 		stmt, syntaxErr := sqlparser.Parse(sql)
 		if syntaxErr != nil {
 			common.Log.Critical("Syntax Error: %v, SQL: %s", syntaxErr, sql)
@@ -95,6 +110,28 @@ func TestRuleImplicitConversion(t *testing.T) {
 			}
 		}
 	}
+	for _, sql := range sqls[1] {
+		stmt, syntaxErr := sqlparser.Parse(sql)
+		if syntaxErr != nil {
+			common.Log.Critical("Syntax Error: %v, SQL: %s", syntaxErr, sql)
+		}
+
+		q := &Query4Audit{Query: sql, Stmt: stmt}
+
+		idxAdvisor, err := NewAdvisor(vEnv, *rEnv, *q)
+		if err != nil {
+			t.Error("NewAdvisor Error: ", err, "SQL: ", sql)
+		}
+
+		if idxAdvisor != nil {
+			rule := idxAdvisor.RuleImplicitConversion()
+			if rule.Item != "OK" {
+				t.Error("Rule not match:", rule, "Expect : OK, SQL:", sql)
+			}
+		}
+	}
+
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 	common.Config.OnlineDSN = dsn
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
@@ -320,6 +357,8 @@ func TestRuleUpdatePrimaryKey(t *testing.T) {
 
 func TestIndexAdvise(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	orgMinCardinality := common.Config.MinCardinality
+	common.Config.MinCardinality = 20
 
 	for _, sql := range common.TestSQLs {
 		stmt, syntaxErr := sqlparser.Parse(sql)
@@ -338,11 +377,12 @@ func TestIndexAdvise(t *testing.T) {
 			if idxAdvisor != nil {
 				rule := idxAdvisor.IndexAdvise().Format()
 				if len(rule) > 0 {
-					pretty.Println(rule)
+					_, _ = pretty.Println(rule)
 				}
 			}
 		}
 	}
+	common.Config.MinCardinality = orgMinCardinality
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
 
@@ -457,7 +497,7 @@ func TestIdxColsTypeCheck(t *testing.T) {
 }
 
 func TestGetRandomIndexSuffix(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	for i := 0; i < 5; i++ {
 		r := getRandomIndexSuffix()
 		if !(strings.HasPrefix(r, "_") && len(r) == 5) {

advisor/rules.go

@@ -471,6 +471,14 @@ func init() {
 			Case:     "CREATE TABLE tbl ( cols ....);",
 			Func:     (*Query4Audit).RuleTooManyFields,
 		},
+		"COL.007": {
+			Item:     "COL.007",
+			Severity: "L3",
+			Summary:  "表中包含有太多的 text/blob 列",
+			Content:  fmt.Sprintf(`表中包含超过%d个的 text/blob 列`, common.Config.MaxTextColsCount),
+			Case:     "CREATE TABLE tbl ( cols ....);",
+			Func:     (*Query4Audit).RuleTooManyFields,
+		},
 		"COL.008": {
 			Item:     "COL.008",
 			Severity: "L1",
@@ -507,10 +515,10 @@ func init() {
 		"COL.012": {
 			Item:     "COL.012",
 			Severity: "L5",
-			Summary:  "BLOB 和 TEXT 类型的字段不可设置为 NULL",
-			Content:  `BLOB 和 TEXT 类型的字段不可设置为 NULL`,
-			Case:     "CREATE TABLE `tbl` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `c` longblob, PRIMARY KEY (`id`));",
-			Func:     (*Query4Audit).RuleCantBeNull,
+			Summary:  "BLOB 和 TEXT 类型的字段不建议设置为 NOT NULL",
+			Content:  `BLOB 和 TEXT 类型的字段无法指定非 NULL 的默认值，如果添加了 NOT NULL 限制，写入数据时又未对该字段指定值可能导致写入失败。`,
+			Case:     "CREATE TABLE `tb`(`c` longblob NOT NULL);",
+			Func:     (*Query4Audit).RuleBLOBNotNull,
 		},
 		"COL.013": {
 			Item:     "COL.013",
@@ -528,12 +536,13 @@ func init() {
 			Case:     "CREATE TABLE `tb2` ( `id` int(11) DEFAULT NULL, `col` char(10) CHARACTER SET utf8 DEFAULT NULL)",
 			Func:     (*Query4Audit).RuleColumnWithCharset,
 		},
+		// https://stackoverflow.com/questions/3466872/why-cant-a-text-column-have-a-default-value-in-mysql
 		"COL.015": {
 			Item:     "COL.015",
 			Severity: "L4",
-			Summary:  "BLOB 类型的字段不可指定默认值",
-			Content:  `BLOB 类型的字段不可指定默认值`,
-			Case:     "CREATE TABLE `tbl` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `c` blob NOT NULL DEFAULT '', PRIMARY KEY (`id`));",
+			Summary:  "TEXT 和 BLOB 类型的字段不可指定非 NULL 的默认值",
+			Content:  `MySQL 数据库中 TEXT 和 BLOB 类型的字段不可指定非 NULL 的默认值。TEXT最大长度为2^16-1个字符，MEDIUMTEXT最大长度为2^32-1个字符，LONGTEXT最大长度为2^64-1个字符。`,
+			Case:     "CREATE TABLE `tbl` (`c` blob DEFAULT NULL);",
 			Func:     (*Query4Audit).RuleBlobDefaultValue,
 		},
 		"COL.016": {
@@ -552,6 +561,22 @@ func init() {
 			Case:     "CREATE TABLE tab (a varchar(3500));",
 			Func:     (*Query4Audit).RuleVarcharLength,
 		},
+		"COL.018": {
+			Item:     "COL.018",
+			Severity: "L1",
+			Summary:  "建表语句中使用了不推荐的字段类型",
+			Content:  "以下字段类型不被推荐使用：" + strings.Join(common.Config.ColumnNotAllowType, ","),
+			Case:     "CREATE TABLE tab (a BOOLEAN);",
+			Func:     (*Query4Audit).RuleColumnNotAllowType,
+		},
+		"COL.019": {
+			Item:     "COL.019",
+			Severity: "L1",
+			Summary:  "不建议使用精度在秒级以下的时间数据类型",
+			Content:  "使用高精度的时间数据类型带来的存储空间消耗相对较大；MySQL 在5.6.4以上才可以支持精确到微秒的时间数据类型，使用时需要考虑版本兼容问题。",
+			Case:     "CREATE TABLE t1 (t TIME(3), dt DATETIME(6));",
+			Func:     (*Query4Audit).RuleTimePrecision,
+		},
 		"DIS.001": {
 			Item:     "DIS.001",
 			Severity: "L1",
@@ -948,6 +973,14 @@ func init() {
 			Case:     "LOAD DATA INFILE 'data.txt' INTO TABLE db2.my_table;",
 			Func:     (*Query4Audit).RuleLoadFile,
 		},
+		"RES.009": {
+			Item:     "RES.009",
+			Severity: "L2",
+			Summary:  "不建议使用连续判断",
+			Content:  "类似这样的 SELECT * FROM tbl WHERE col = col = 'abc' 语句可能是书写错误，您可能想表达的含义是 col = 'abc'。如果确实是业务需求建议修改为 col = col and col = 'abc'。",
+			Case:     "SELECT * FROM tbl WHERE col = col = 'abc'",
+			Func:     (*Query4Audit).RuleMultiCompare,
+		},
 		"SEC.001": {
 			Item:     "SEC.001",
 			Severity: "L0",
@@ -1068,7 +1101,7 @@ func init() {
 			Item:     "TBL.002",
 			Severity: "L4",
 			Summary:  "请为表选择合适的存储引擎",
-			Content:  `建表或修改表的存储引擎时建议使用推荐的存储引擎，如：` + strings.Join(common.Config.TableAllowEngines, ","),
+			Content:  `建表或修改表的存储引擎时建议使用推荐的存储引擎，如：` + strings.Join(common.Config.AllowEngines, ","),
 			Case:     "create table test(`id` int(11) NOT NULL AUTO_INCREMENT)",
 			Func:     (*Query4Audit).RuleAllowEngine,
 		},
@@ -1092,7 +1125,7 @@ func init() {
 			Item:     "TBL.005",
 			Severity: "L4",
 			Summary:  "请使用推荐的字符集",
-			Content:  `表字符集只允许设置为` + strings.Join(common.Config.TableAllowCharsets, ","),
+			Content:  `表字符集只允许设置为'` + strings.Join(common.Config.AllowCharsets, ",") + "'",
 			Case:     "CREATE TABLE tbl (a int) DEFAULT CHARSET = latin1;",
 			Func:     (*Query4Audit).RuleTableCharsetCheck,
 		},
@@ -1112,6 +1145,14 @@ func init() {
 			Case:     "CREATE TEMPORARY TABLE `work` (`time` time DEFAULT NULL) ENGINE=InnoDB;",
 			Func:     (*Query4Audit).RuleForbiddenTempTable,
 		},
+		"TBL.008": {
+			Item:     "TBL.008",
+			Severity: "L4",
+			Summary:  "请使用推荐的COLLATE",
+			Content:  `COLLATE 只允许设置为'` + strings.Join(common.Config.AllowCollates, ",") + "'",
+			Case:     "CREATE TABLE tbl (a int) DEFAULT COLLATE = latin1_bin;",
+			Func:     (*Query4Audit).RuleTableCharsetCheck,
+		},
 	}
 }
 

advisor/rules_test.go

@@ -17,6 +17,7 @@
 package advisor
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/XiaoMi/soar/common"
@@ -24,6 +25,11 @@ import (
 
 func TestListTestSQLs(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	for _, sql := range common.TestSQLs {
+		if !strings.HasSuffix(sql, ";") {
+			t.Errorf("%s should end with ';'", sql)
+		}
+	}
 	err := common.GoldenDiff(func() { ListTestSQLs() }, t.Name(), update)
 	if nil != err {
 		t.Fatal(err)

advisor/testdata/TestListHeuristicRules.golden

@@ -409,6 +409,16 @@ CREATE TABLE tbl (col int) ENGINE=InnoDB;
 * **Content**:表中包含有太多的列
 * **Case**:
 
+```sql
+CREATE TABLE tbl ( cols ....);
+```
+## 表中包含有太多的 text/blob 列
+
+* **Item**:COL.007
+* **Severity**:L3
+* **Content**:表中包含超过2个的 text/blob 列
+* **Case**:
+
 ```sql
 CREATE TABLE tbl ( cols ....);
 ```
@@ -452,15 +462,15 @@ create table tab1(status ENUM('new','in progress','fixed'))
 ```sql
 select c1,c2,c3 from tbl where c4 is null or c4 <> 1
 ```
-## BLOB 和 TEXT 类型的字段不可设置为 NULL
+## BLOB 和 TEXT 类型的字段不建议设置为 NOT NULL
 
 * **Item**:COL.012
 * **Severity**:L5
-* **Content**:BLOB 和 TEXT 类型的字段不可设置为 NULL
+* **Content**:BLOB 和 TEXT 类型的字段无法指定非 NULL 的默认值，如果添加了 NOT NULL 限制，写入数据时又未对该字段指定值可能导致写入失败。
 * **Case**:
 
 ```sql
-CREATE TABLE `tbl` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `c` longblob, PRIMARY KEY (`id`));
+CREATE TABLE `tb`(`c` longblob NOT NULL);
 ```
 ## TIMESTAMP 类型未设置默认值
 
@@ -482,15 +492,15 @@ CREATE TABLE tbl( `id` bigint not null, `create_time` timestamp);
 ```sql
 CREATE TABLE `tb2` ( `id` int(11) DEFAULT NULL, `col` char(10) CHARACTER SET utf8 DEFAULT NULL)
 ```
-## BLOB 类型的字段不可指定默认值
+## TEXT 和 BLOB 类型的字段不可指定非 NULL 的默认值
 
 * **Item**:COL.015
 * **Severity**:L4
-* **Content**:BLOB 类型的字段不可指定默认值
+* **Content**:MySQL 数据库中 TEXT 和 BLOB 类型的字段不可指定非 NULL 的默认值。TEXT最大长度为2^16-1个字符，MEDIUMTEXT最大长度为2^32-1个字符，LONGTEXT最大长度为2^64-1个字符。
 * **Case**:
 
 ```sql
-CREATE TABLE `tbl` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `c` blob NOT NULL DEFAULT '', PRIMARY KEY (`id`));
+CREATE TABLE `tbl` (`c` blob DEFAULT NULL);
 ```
 ## 整型定义建议采用 INT(10) 或 BIGINT(20)
 
@@ -512,6 +522,26 @@ CREATE TABLE tab (a INT(1));
 ```sql
 CREATE TABLE tab (a varchar(3500));
 ```
+## 建表语句中使用了不推荐的字段类型
+
+* **Item**:COL.018
+* **Severity**:L1
+* **Content**:以下字段类型不被推荐使用：boolean
+* **Case**:
+
+```sql
+CREATE TABLE tab (a BOOLEAN);
+```
+## 不建议使用精度在秒级以下的时间数据类型
+
+* **Item**:COL.019
+* **Severity**:L1
+* **Content**:使用高精度的时间数据类型带来的存储空间消耗相对较大；MySQL 在5.6.4以上才可以支持精确到微秒的时间数据类型，使用时需要考虑版本兼容问题。
+* **Case**:
+
+```sql
+CREATE TABLE t1 (t TIME(3), dt DATETIME(6));
+```
 ## 消除不必要的 DISTINCT 条件
 
 * **Item**:DIS.001
@@ -1002,6 +1032,16 @@ select * from tbl where 1 = 1;
 ```sql
 LOAD DATA INFILE 'data.txt' INTO TABLE db2.my_table;
 ```
+## 不建议使用连续判断
+
+* **Item**:RES.009
+* **Severity**:L2
+* **Content**:类似这样的 SELECT \* FROM tbl WHERE col = col = 'abc' 语句可能是书写错误，您可能想表达的含义是 col = 'abc'。如果确实是业务需求建议修改为 col = col and col = 'abc'。
+* **Case**:
+
+```sql
+SELECT * FROM tbl WHERE col = col = 'abc'
+```
 ## 请谨慎使用TRUNCATE操作
 
 * **Item**:SEC.001
@@ -1176,7 +1216,7 @@ CREATE TABLE tbl (a int) AUTO_INCREMENT = 10;
 
 * **Item**:TBL.005
 * **Severity**:L4
-* **Content**:表字符集只允许设置为utf8,utf8mb4
+* **Content**:表字符集只允许设置为'utf8,utf8mb4'
 * **Case**:
 
 ```sql
@@ -1202,3 +1242,13 @@ create view v_today (today) AS SELECT CURRENT_DATE;
 ```sql
 CREATE TEMPORARY TABLE `work` (`time` time DEFAULT NULL) ENGINE=InnoDB;
 ```
+## 请使用推荐的COLLATE
+
+* **Item**:TBL.008
+* **Severity**:L4
+* **Content**:COLLATE 只允许设置为''
+* **Case**:
+
+```sql
+CREATE TABLE tbl (a int) DEFAULT COLLATE = latin1_bin;
+```

advisor/testdata/TestListTestSQLs.golden

@@ -45,7 +45,7 @@ SELECT country_id, last_update FROM city NATURAL JOIN country;
 SELECT country_id, last_update FROM city NATURAL LEFT JOIN country;
 SELECT country_id, last_update FROM city NATURAL RIGHT JOIN country;
 SELECT a.country_id, a.last_update FROM city a STRAIGHT_JOIN country b ON a.country_id=b.country_id;
-SELECT d.deptno,d.dname,d.loc FROM scott.dept d WHERE d.deptno IN  (SELECT e.deptno FROM scott.emp e);
+SELECT a.address, a.postal_code FROM sakila.address a WHERE a.city_id IN  (SELECT c.city_id FROM sakila.city c);
 SELECT visitor_id, url FROM (SELECT id FROM log WHERE ip="123.45.67.89" order by ts desc limit 50, 10) I JOIN log ON (I.id=log.id) JOIN url ON (url.id=log.url_id) order by TS desc;
 DELETE city, country FROM city INNER JOIN country using (country_id) WHERE city.city_id = 1;
 DELETE city FROM city LEFT JOIN country ON city.country_id = country.country_id WHERE country.country IS NULL;
@@ -79,5 +79,7 @@ SELECT description FROM film WHERE description IN('NEWS','asd') GROUP BY descrip
 alter table address add index idx_city_id(city_id);
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
-SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
 create table hello.t (id int unsigned);
+select * from tb where data >= '';
+alter table tb alter column id drop default;

advisor/testdata/TestMergeConflictHeuristicRules.golden

@@ -38,16 +38,19 @@ advisor.Rule{Item:"COL.003", Severity:"L2", Summary:"建议修改自增 ID 为
 advisor.Rule{Item:"COL.004", Severity:"L1", Summary:"请为列添加默认值", Content:"请为列添加默认值，如果是 ALTER 操作，请不要忘记将原字段的默认值写上。字段无默认值，当表较大时无法在线变更表结构。", Case:"CREATE TABLE tbl (col int) ENGINE=InnoDB;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.005", Severity:"L1", Summary:"列未添加注释", Content:"建议对表中每个列添加注释，来明确每个列在表中的含义及作用。", Case:"CREATE TABLE tbl (col int) ENGINE=InnoDB;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.006", Severity:"L3", Summary:"表中包含有太多的列", Content:"表中包含有太多的列", Case:"CREATE TABLE tbl ( cols ....);", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
+advisor.Rule{Item:"COL.007", Severity:"L3", Summary:"表中包含有太多的 text/blob 列", Content:"表中包含超过2个的 text/blob 列", Case:"CREATE TABLE tbl ( cols ....);", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.008", Severity:"L1", Summary:"可使用 VARCHAR 代替 CHAR， VARBINARY 代替 BINARY", Content:"为首先变长字段存储空间小，可以节省存储空间。其次对于查询来说，在一个相对较小的字段内搜索效率显然要高些。", Case:"create table t1(id int,name char(20),last_time date)", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.009", Severity:"L2", Summary:"建议使用精确的数据类型", Content:"实际上，任何使用 FLOAT, REAL 或 DOUBLE PRECISION 数据类型的设计都有可能是反模式。大多数应用程序使用的浮点数的取值范围并不需要达到IEEE 754标准所定义的最大/最小区间。在计算总量时，非精确浮点数所积累的影响是严重的。使用 SQL 中的 NUMERIC 或 DECIMAL 类型来代替 FLOAT 及其类似的数据类型进行固定精度的小数存储。这些数据类型精确地根据您定义这一列时指定的精度来存储数据。尽可能不要使用浮点数。", Case:"CREATE TABLE tab2 (p_id  BIGINT UNSIGNED NOT NULL,a_id  BIGINT UNSIGNED NOT NULL,hours float not null,PRIMARY KEY (p_id, a_id))", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.010", Severity:"L2", Summary:"不建议使用 ENUM 数据类型", Content:"ENUM 定义了列中值的类型，使用字符串表示 ENUM 里的值时，实际存储在列中的数据是这些值在定义时的序数。因此，这列的数据是字节对齐的，当您进行一次排序查询时，结果是按照实际存储的序数值排序的，而不是按字符串值的字母顺序排序的。这可能不是您所希望的。没有什么语法支持从 ENUM 或者 check 约束中添加或删除一个值；您只能使用一个新的集合重新定义这一列。如果您打算废弃一个选项，您可能会为历史数据而烦恼。作为一种策略，改变元数据——也就是说，改变表和列的定义——应该是不常见的，并且要注意测试和质量保证。有一个更好的解决方案来约束一列中的可选值:创建一张检查表，每一行包含一个允许在列中出现的候选值；然后在引用新表的旧表上声明一个外键约束。", Case:"create table tab1(status ENUM('new','in progress','fixed'))", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.011", Severity:"L0", Summary:"当需要唯一约束时才使用 NULL，仅当列不能有缺失值时才使用 NOT NULL", Content:"NULL 和0是不同的，10乘以 NULL 还是 NULL。NULL 和空字符串是不一样的。将一个字符串和标准 SQL 中的 NULL 联合起来的结果还是 NULL。NULL 和 FALSE 也是不同的。AND、OR 和 NOT 这三个布尔操作如果涉及 NULL，其结果也让很多人感到困惑。当您将一列声明为 NOT NULL 时，也就是说这列中的每一个值都必须存在且是有意义的。使用 NULL 来表示任意类型不存在的空值。 当您将一列声明为 NOT NULL 时，也就是说这列中的每一个值都必须存在且是有意义的。", Case:"select c1,c2,c3 from tbl where c4 is null or c4 <> 1", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
-advisor.Rule{Item:"COL.012", Severity:"L5", Summary:"BLOB 和 TEXT 类型的字段不可设置为 NULL", Content:"BLOB 和 TEXT 类型的字段不可设置为 NULL", Case:"CREATE TABLE `tbl` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `c` longblob, PRIMARY KEY (`id`));", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
+advisor.Rule{Item:"COL.012", Severity:"L5", Summary:"BLOB 和 TEXT 类型的字段不建议设置为 NOT NULL", Content:"BLOB 和 TEXT 类型的字段无法指定非 NULL 的默认值，如果添加了 NOT NULL 限制，写入数据时又未对该字段指定值可能导致写入失败。", Case:"CREATE TABLE `tb`(`c` longblob NOT NULL);", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.013", Severity:"L4", Summary:"TIMESTAMP 类型未设置默认值", Content:"TIMESTAMP 类型未设置默认值", Case:"CREATE TABLE tbl( `id` bigint not null, `create_time` timestamp);", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.014", Severity:"L5", Summary:"为列指定了字符集", Content:"建议列与表使用同一个字符集，不要单独指定列的字符集。", Case:"CREATE TABLE `tb2` ( `id` int(11) DEFAULT NULL, `col` char(10) CHARACTER SET utf8 DEFAULT NULL)", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
-advisor.Rule{Item:"COL.015", Severity:"L4", Summary:"BLOB 类型的字段不可指定默认值", Content:"BLOB 类型的字段不可指定默认值", Case:"CREATE TABLE `tbl` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `c` blob NOT NULL DEFAULT '', PRIMARY KEY (`id`));", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
+advisor.Rule{Item:"COL.015", Severity:"L4", Summary:"TEXT 和 BLOB 类型的字段不可指定非 NULL 的默认值", Content:"MySQL 数据库中 TEXT 和 BLOB 类型的字段不可指定非 NULL 的默认值。TEXT最大长度为2^16-1个字符，MEDIUMTEXT最大长度为2^32-1个字符，LONGTEXT最大长度为2^64-1个字符。", Case:"CREATE TABLE `tbl` (`c` blob DEFAULT NULL);", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.016", Severity:"L1", Summary:"整型定义建议采用 INT(10) 或 BIGINT(20)", Content:"INT(M) 在 integer 数据类型中，M 表示最大显示宽度。 在 INT(M) 中，M 的值跟 INT(M) 所占多少存储空间并无任何关系。 INT(3)、INT(4)、INT(8) 在磁盘上都是占用 4 bytes 的存储空间。", Case:"CREATE TABLE tab (a INT(1));", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.017", Severity:"L2", Summary:"VARCHAR 定义长度过长", Content:"varchar 是可变长字符串，不预先分配存储空间，长度不要超过1024，如果存储长度过长 MySQL 将定义字段类型为 text，独立出来一张表，用主键来对应，避免影响其它字段索引效率。", Case:"CREATE TABLE tab (a varchar(3500));", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
+advisor.Rule{Item:"COL.018", Severity:"L1", Summary:"建表语句中使用了不推荐的字段类型", Content:"以下字段类型不被推荐使用：boolean", Case:"CREATE TABLE tab (a BOOLEAN);", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
+advisor.Rule{Item:"COL.019", Severity:"L1", Summary:"不建议使用精度在秒级以下的时间数据类型", Content:"使用高精度的时间数据类型带来的存储空间消耗相对较大；MySQL 在5.6.4以上才可以支持精确到微秒的时间数据类型，使用时需要考虑版本兼容问题。", Case:"CREATE TABLE t1 (t TIME(3), dt DATETIME(6));", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"DIS.001", Severity:"L1", Summary:"消除不必要的 DISTINCT 条件", Content:"太多DISTINCT条件是复杂的裹脚布式查询的症状。考虑将复杂查询分解成许多简单的查询，并减少DISTINCT条件的数量。如果主键列是列的结果集的一部分，则DISTINCT条件可能没有影响。", Case:"SELECT DISTINCT c.c_id,count(DISTINCT c.c_name),count(DISTINCT c.c_e),count(DISTINCT c.c_n),count(DISTINCT c.c_me),c.c_d FROM (select distinct id, name from B) as e WHERE e.country_id = c.country_id", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"DIS.002", Severity:"L3", Summary:"COUNT(DISTINCT) 多列时结果可能和你预想的不同", Content:"COUNT(DISTINCT col) 计算该列除NULL之外的不重复行数，注意 COUNT(DISTINCT col, col2) 如果其中一列全为 NULL 那么即使另一列有不同的值，也返回0。", Case:"SELECT COUNT(DISTINCT col, col2) FROM tbl;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"DIS.003", Severity:"L3", Summary:"DISTINCT * 对有主键的表没有意义", Content:"当表已经有主键时，对所有列进行 DISTINCT 的输出结果与不进行 DISTINCT 操作的结果相同，请不要画蛇添足。", Case:"SELECT DISTINCT * FROM film;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
@@ -95,6 +98,7 @@ advisor.Rule{Item:"RES.005", Severity:"L4", Summary:"UPDATE 语句可能存在
 advisor.Rule{Item:"RES.006", Severity:"L4", Summary:"永远不真的比较条件", Content:"查询条件永远非真，如果该条件出现在 where 中可能导致查询无匹配到的结果。", Case:"select * from tbl where 1 != 1;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"RES.007", Severity:"L4", Summary:"永远为真的比较条件", Content:"查询条件永远为真，可能导致 WHERE 条件失效进行全表查询。", Case:"select * from tbl where 1 = 1;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"RES.008", Severity:"L2", Summary:"不建议使用LOAD DATA/SELECT ... INTO OUTFILE", Content:"SELECT INTO OUTFILE 需要授予 FILE 权限，这通过会引入安全问题。LOAD DATA 虽然可以提高数据导入速度，但同时也可能导致从库同步延迟过大。", Case:"LOAD DATA INFILE 'data.txt' INTO TABLE db2.my_table;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
+advisor.Rule{Item:"RES.009", Severity:"L2", Summary:"不建议使用连续判断", Content:"类似这样的 SELECT * FROM tbl WHERE col = col = 'abc' 语句可能是书写错误，您可能想表达的含义是 col = 'abc'。如果确实是业务需求建议修改为 col = col and col = 'abc'。", Case:"SELECT * FROM tbl WHERE col = col = 'abc'", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"SEC.001", Severity:"L0", Summary:"请谨慎使用TRUNCATE操作", Content:"一般来说想清空一张表最快速的做法就是使用TRUNCATE TABLE tbl_name;语句。但TRUNCATE操作也并非是毫无代价的，TRUNCATE TABLE无法返回被删除的准确行数，如果需要返回被删除的行数建议使用DELETE语法。TRUNCATE 操作还会重置 AUTO_INCREMENT，如果不想重置该值建议使用 DELETE FROM tbl_name WHERE 1;替代。TRUNCATE 操作会对数据字典添加源数据锁(MDL)，当一次需要 TRUNCATE 很多表时会影响整个实例的所有请求，因此如果要 TRUNCATE 多个表建议用 DROP+CREATE 的方式以减少锁时长。", Case:"TRUNCATE TABLE tbl_name", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"SEC.002", Severity:"L0", Summary:"不使用明文存储密码", Content:"使用明文存储密码或者使用明文在网络上传递密码都是不安全的。如果攻击者能够截获您用来插入密码的SQL语句，他们就能直接读到密码。另外，将用户输入的字符串以明文的形式插入到纯SQL语句中，也会让攻击者发现它。如果您能够读取密码，黑客也可以。解决方案是使用单向哈希函数对原始密码进行加密编码。哈希是指将输入字符串转化成另一个新的、不可识别的字符串的函数。对密码加密表达式加点随机串来防御“字典攻击”。不要将明文密码输入到SQL查询语句中。在应用程序代码中计算哈希串，只在SQL查询中使用哈希串。", Case:"create table test(id int,name varchar(20) not null,password varchar(200)not null)", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"SEC.003", Severity:"L0", Summary:"使用DELETE/DROP/TRUNCATE等操作时注意备份", Content:"在执行高危操作之前对数据进行备份是十分有必要的。", Case:"delete from table where col = 'condition'", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
@@ -111,6 +115,7 @@ advisor.Rule{Item:"TBL.001", Severity:"L4", Summary:"不建议使用分区表",
 advisor.Rule{Item:"TBL.002", Severity:"L4", Summary:"请为表选择合适的存储引擎", Content:"建表或修改表的存储引擎时建议使用推荐的存储引擎，如：innodb", Case:"create table test(`id` int(11) NOT NULL AUTO_INCREMENT)", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"TBL.003", Severity:"L8", Summary:"以DUAL命名的表在数据库中有特殊含义", Content:"DUAL表为虚拟表，不需要创建即可使用，也不建议服务以DUAL命名表。", Case:"create table dual(id int, primary key (id));", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"TBL.004", Severity:"L2", Summary:"表的初始AUTO_INCREMENT值不为0", Content:"AUTO_INCREMENT不为0会导致数据空洞。", Case:"CREATE TABLE tbl (a int) AUTO_INCREMENT = 10;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
-advisor.Rule{Item:"TBL.005", Severity:"L4", Summary:"请使用推荐的字符集", Content:"表字符集只允许设置为utf8,utf8mb4", Case:"CREATE TABLE tbl (a int) DEFAULT CHARSET = latin1;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
+advisor.Rule{Item:"TBL.005", Severity:"L4", Summary:"请使用推荐的字符集", Content:"表字符集只允许设置为'utf8,utf8mb4'", Case:"CREATE TABLE tbl (a int) DEFAULT CHARSET = latin1;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"TBL.006", Severity:"L1", Summary:"不建议使用视图", Content:"不建议使用视图", Case:"create view v_today (today) AS SELECT CURRENT_DATE;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"TBL.007", Severity:"L1", Summary:"不建议使用临时表", Content:"不建议使用临时表", Case:"CREATE TEMPORARY TABLE `work` (`time` time DEFAULT NULL) ENGINE=InnoDB;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
+advisor.Rule{Item:"TBL.008", Severity:"L4", Summary:"请使用推荐的COLLATE", Content:"COLLATE 只允许设置为''", Case:"CREATE TABLE tbl (a int) DEFAULT COLLATE = latin1_bin;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}

ast/meta_test.go

@@ -17,6 +17,7 @@
 package ast
 
 import (
+	"flag"
 	"fmt"
 	"testing"
 
@@ -26,6 +27,23 @@ import (
 	"vitess.io/vitess/go/vt/sqlparser"
 )
 
+var update = flag.Bool("update", false, "update .golden files")
+
+func TestMain(m *testing.M) {
+	// 初始化 init
+	common.BaseDir = common.DevPath
+	err := common.ParseConfig("")
+	common.LogIfError(err, "init ParseConfig")
+	common.Log.Debug("ast_test init")
+
+	// 分割线
+	flag.Parse()
+	m.Run()
+
+	// 环境清理
+	//
+}
+
 func TestGetTableFromExprs(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	tbExprs := sqlparser.TableExprs{

ast/pretty.go

@@ -32,7 +32,7 @@ func Pretty(sql string, method string) (output string) {
 	// 超出 Config.MaxPrettySQLLength 长度的 SQL 会对其指纹进行 pretty
 	if len(sql) > common.Config.MaxPrettySQLLength {
 		fingerprint := query.Fingerprint(sql)
-		// 超出 Config.MaxFpPrettySqlLength 长度的指纹不会进行pretty
+		// 超出 Config.MaxPrettySQLLength 长度的指纹不会进行pretty
 		if len(fingerprint) > common.Config.MaxPrettySQLLength {
 			return sql
 		}

ast/pretty_test.go

@@ -17,7 +17,6 @@
 package ast
 
 import (
-	"flag"
 	"fmt"
 	"testing"
 
@@ -26,8 +25,6 @@ import (
 	"vitess.io/vitess/go/vt/sqlparser"
 )
 
-var update = flag.Bool("update", false, "update .golden files")
-
 var TestSqlsPretty = []string{
 	"select sourcetable, if(f.lastcontent = ?, f.lastupdate, f.lastcontent) as lastactivity, f.totalcount as activity, type.class as type, (f.nodeoptions & ?) as nounsubscribe from node as f inner join contenttype as type on type.contenttypeid = f.contenttypeid inner join subscribed as sd on sd.did = f.nodeid and sd.userid = ? union all select f.name as title, f.userid as keyval, ? as sourcetable, ifnull(f.lastpost, f.joindate) as lastactivity, f.posts as activity, ? as type, ? as nounsubscribe from user as f inner join userlist as ul on ul.relationid = f.userid and ul.userid = ? where ul.type = ? and ul.aq = ? order by title limit ?",
 	"administrator command: Init DB",

ast/testdata/TestCompress.golden

@@ -92,8 +92,8 @@ SELECT country_id, last_update FROM city NATURAL RIGHT JOIN country;
 SELECT country_id, last_update FROM city NATURAL RIGHT JOIN country;
 SELECT a.country_id, a.last_update FROM city a STRAIGHT_JOIN country b ON a.country_id=b.country_id;
 SELECT a.country_id, a.last_update FROM city a STRAIGHT_JOIN country b ON a.country_id=b.country_id;
-SELECT d.deptno,d.dname,d.loc FROM scott.dept d WHERE d.deptno IN  (SELECT e.deptno FROM scott.emp e);
-SELECT d.deptno,d.dname,d.loc FROM scott.dept d WHERE d.deptno IN (SELECT e.deptno FROM scott.emp e);
+SELECT a.address, a.postal_code FROM sakila.address a WHERE a.city_id IN  (SELECT c.city_id FROM sakila.city c);
+SELECT a.address, a.postal_code FROM sakila.address a WHERE a.city_id IN (SELECT c.city_id FROM sakila.city c);
 SELECT visitor_id, url FROM (SELECT id FROM log WHERE ip="123.45.67.89" order by ts desc limit 50, 10) I JOIN log ON (I.id=log.id) JOIN url ON (url.id=log.url_id) order by TS desc;
 SELECT visitor_id, url FROM (SELECT id FROM log WHERE ip="123.45.67.89" order by ts desc limit 50, 10) I JOIN log ON (I.id=log.id) JOIN url ON (url.id=log.url_id) order by TS desc;
 DELETE city, country FROM city INNER JOIN country using (country_id) WHERE city.city_id = 1;
@@ -160,7 +160,11 @@ alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
-SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
-SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
+SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
 create table hello.t (id int unsigned);
 create table hello.t (id int unsigned);
+select * from tb where data >= '';
+select * from tb where data >= '';
+alter table tb alter column id drop default;
+alter table tb alter column id drop default;

ast/testdata/TestFormat.golden

@@ -444,18 +444,18 @@ SELECT
   a. country_id, a. last_update  
 FROM  
   city  a  STRAIGHT_JOIN  country  b  ON  a. country_id= b. country_id;
-SELECT d.deptno,d.dname,d.loc FROM scott.dept d WHERE d.deptno IN  (SELECT e.deptno FROM scott.emp e);
+SELECT a.address, a.postal_code FROM sakila.address a WHERE a.city_id IN  (SELECT c.city_id FROM sakila.city c);
 
 SELECT  
-  d. deptno, d. dname, d. loc  
+  a. address, a. postal_code  
 FROM  
-  scott. dept  d  
+  sakila. address  a  
 WHERE  
-  d. deptno  IN  (
+  a. city_id  IN  (
 SELECT  
-  e. deptno  
+  c. city_id  
 FROM  
-  scott. emp  e);
+  sakila. city  c);
 SELECT visitor_id, url FROM (SELECT id FROM log WHERE ip="123.45.67.89" order by ts desc limit 50, 10) I JOIN log ON (I.id=log.id) JOIN url ON (url.id=log.url_id) order by TS desc;
 
 SELECT  
@@ -846,7 +846,7 @@ ADD
               ADD  
           index  `idx_store_film` (
             `store_id`, `film_id`);
-SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
 
 SELECT  
   DATE_FORMAT( t. atm, '%Y-%m-%d'
@@ -864,6 +864,20 @@ COUNT( DISTINCT  (
 ) 
 ORDER BY  
   DATE_FORMAT( t. atm, '%Y-%m-%d'
-)
+);
 create table hello.t (id int unsigned);
 create  table  hello. t  (id  int  unsigned);
+select * from tb where data >= '';
+
+SELECT  
+  * 
+FROM  
+  tb  
+WHERE  
+  data  >= '';
+alter table tb alter column id drop default;
+
+ALTER TABLE  
+  tb  alter  column  id  
+DROP  
+  DEFAULT;

ast/testdata/TestPretty.golden

@@ -1088,18 +1088,18 @@ SELECT
   a. country_id, a. last_update  
 FROM  
   city  a  STRAIGHT_JOIN  country  b  ON  a. country_id= b. country_id;
-SELECT d.deptno,d.dname,d.loc FROM scott.dept d WHERE d.deptno IN  (SELECT e.deptno FROM scott.emp e);
+SELECT a.address, a.postal_code FROM sakila.address a WHERE a.city_id IN  (SELECT c.city_id FROM sakila.city c);
 
 SELECT  
-  d. deptno, d. dname, d. loc  
+  a. address, a. postal_code  
 FROM  
-  scott. dept  d  
+  sakila. address  a  
 WHERE  
-  d. deptno  IN  (
+  a. city_id  IN  (
 SELECT  
-  e. deptno  
+  c. city_id  
 FROM  
-  scott. emp  e);
+  sakila. city  c);
 SELECT visitor_id, url FROM (SELECT id FROM log WHERE ip="123.45.67.89" order by ts desc limit 50, 10) I JOIN log ON (I.id=log.id) JOIN url ON (url.id=log.url_id) order by TS desc;
 
 SELECT  
@@ -1490,7 +1490,7 @@ ADD
               ADD  
           index  `idx_store_film` (
             `store_id`, `film_id`);
-SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
 
 SELECT  
   DATE_FORMAT( t. atm, '%Y-%m-%d'
@@ -1508,6 +1508,20 @@ COUNT( DISTINCT  (
 ) 
 ORDER BY  
   DATE_FORMAT( t. atm, '%Y-%m-%d'
-)
+);
 create table hello.t (id int unsigned);
 create  table  hello. t  (id  int  unsigned);
+select * from tb where data >= '';
+
+SELECT  
+  * 
+FROM  
+  tb  
+WHERE  
+  data  >= '';
+alter table tb alter column id drop default;
+
+ALTER TABLE  
+  tb  alter  column  id  
+DROP  
+  DEFAULT;

ast/testdata/TestPrintPrettyStmtNode.golden

+[]ast.StmtNode{
+    &ast.SelectStmt{
+        dmlNode: ast.dmlNode{
+            stmtNode: ast.stmtNode{
+                node: ast.node{text:"select 1"},
+            },
+        },
+        resultSetNode:  ast.resultSetNode{},
+        SelectStmtOpts: &ast.SelectStmtOpts{
+            Distinct:      false,
+            SQLCache:      true,
+            CalcFoundRows: false,
+            StraightJoin:  false,
+            Priority:      0,
+            TableHints:    nil,
+        },
+        Distinct: false,
+        From:     (*ast.TableRefsClause)(nil),
+        Where:    nil,
+        Fields:   &ast.FieldList{
+            node:   ast.node{},
+            Fields: {
+                &ast.SelectField{
+                    node:     ast.node{text:"1"},
+                    Offset:   7,
+                    WildCard: (*ast.WildCardField)(nil),
+                    Expr:     &driver.ValueExpr{
+                        TexprNode: ast.exprNode{
+                            node: ast.node{},
+                            Type: types.FieldType{
+                                Tp:      0x8,
+                                Flag:    0x80,
+                                Flen:    1,
+                                Decimal: 0,
+                                Charset: "binary",
+                                Collate: "binary",
+                                Elems:   nil,
+                            },
+                            flag: 0x0,
+                        },
+                        Datum: types.Datum{
+                            k:         0x1,
+                            collation: 0x0,
+                            decimal:   0x0,
+                            length:    0x0,
+                            i:         1,
+                            b:         nil,
+                            x:         nil,
+                        },
+                        projectionOffset: -1,
+                    },
+                    AsName:    model.CIStr{},
+                    Auxiliary: false,
+                },
+            },
+        },
+        GroupBy:              (*ast.GroupByClause)(nil),
+        Having:               (*ast.HavingClause)(nil),
+        WindowSpecs:          nil,
+        OrderBy:              (*ast.OrderByClause)(nil),
+        Limit:                (*ast.Limit)(nil),
+        LockTp:               0,
+        TableHints:           nil,
+        IsAfterUnionDistinct: false,
+        IsInBraces:           false,
+    },
+}

ast/testdata/TestPrintPrettyVitessStmtNode.golden

+&sqlparser.Select{
+    Cache:       "",
+    Comments:    nil,
+    Distinct:    "",
+    Hints:       "",
+    SelectExprs: {
+        &sqlparser.AliasedExpr{
+            Expr: &sqlparser.SQLVal{
+                Type: 1,
+                Val:  {0x31},
+            },
+            As: sqlparser.ColIdent{},
+        },
+    },
+    From: {
+        &sqlparser.AliasedTableExpr{
+            Expr: sqlparser.TableName{
+                Name:      sqlparser.TableIdent{v:"dual"},
+                Qualifier: sqlparser.TableIdent{},
+            },
+            Partitions: nil,
+            As:         sqlparser.TableIdent{},
+            Hints:      (*sqlparser.IndexHints)(nil),
+        },
+    },
+    Where:   (*sqlparser.Where)(nil),
+    GroupBy: nil,
+    Having:  (*sqlparser.Where)(nil),
+    OrderBy: nil,
+    Limit:   (*sqlparser.Limit)(nil),
+    Lock:    "",
+}

ast/testdata/TestStmtNode2JSON.golden

+[
+  {
+    "SQLCache": true,
+    "CalcFoundRows": false,
+    "StraightJoin": false,
+    "Priority": 0,
+    "Distinct": false,
+    "From": null,
+    "Where": null,
+    "Fields": {
+      "Fields": [
+        {
+          "Offset": 7,
+          "WildCard": null,
+          "Expr": {
+            "Type": {
+              "Tp": 8,
+              "Flag": 128,
+              "Flen": 1,
+              "Decimal": 0,
+              "Charset": "binary",
+              "Collate": "binary",
+              "Elems": null
+            }
+          },
+          "AsName": {
+            "O": "",
+            "L": ""
+          },
+          "Auxiliary": false
+        }
+      ]
+    },
+    "GroupBy": null,
+    "Having": null,
+    "WindowSpecs": null,
+    "OrderBy": null,
+    "Limit": null,
+    "LockTp": 0,
+    "TableHints": null,
+    "IsAfterUnionDistinct": false,
+    "IsInBraces": false
+  }
+]

ast/testdata/TestTokenize.golden

@@ -5,9 +5,9 @@ SELECT * FROM film WHERE length IS NULL;
 SELECT * FROM film HAVING title = 'abc';
 [{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 HAVING  0} {1 title  0} {7 = 0} {0   0} {2 'abc' 0} {7 ; 0}]
 SELECT * FROM sakila.film WHERE length >= 60;
-[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 sakila. 0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {7 = 0} {0   0} {10 60; 0}]
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 sakila. 0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 >= 0} {0   0} {10 60; 0}]
 SELECT * FROM sakila.film WHERE length >= '60';
-[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 sakila. 0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {7 = 0} {0   0} {2 '60' 0} {7 ; 0}]
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 sakila. 0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 >= 0} {0   0} {2 '60' 0} {7 ; 0}]
 SELECT * FROM film WHERE length BETWEEN 60 AND 84;
 [{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {1 BETWEEN  0} {10 60  0} {6 AND  0} {10 84; 0}]
 SELECT * FROM film WHERE title LIKE 'AIR%';
@@ -25,7 +25,7 @@ SELECT * FROM film WHERE length > 100 and language_id < 10;
 SELECT release_year, sum(length) FROM film WHERE length = 123 AND language_id = 1 GROUP BY release_year;
 [{5 SELECT  0} {1 release_year, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 123  0} {6 AND  0} {1 language_id  0} {7 = 0} {0   0} {10 1  0} {5 GROUP BY  0} {1 release_year; 0}]
 SELECT release_year, sum(length) FROM film WHERE length >= 123 GROUP BY release_year;
-[{5 SELECT  0} {1 release_year, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {7 = 0} {0   0} {10 123  0} {5 GROUP BY  0} {1 release_year; 0}]
+[{5 SELECT  0} {1 release_year, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 >= 0} {0   0} {10 123  0} {5 GROUP BY  0} {1 release_year; 0}]
 SELECT release_year, language_id, sum(length) FROM film GROUP BY release_year, language_id;
 [{5 SELECT  0} {1 release_year, 0} {0   0} {1 language_id, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 GROUP BY  0} {1 release_year, 0} {0   0} {1 language_id; 0}]
 SELECT release_year, sum(length) FROM film WHERE length = 123 GROUP BY release_year,(length+language_id);
@@ -57,7 +57,7 @@ SELECT * FROM film WHERE length < 100 ORDER BY length LIMIT 10;
 SELECT * FROM customer WHERE address_id in (224,510) ORDER BY last_name;
 [{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 customer  0} {5 WHERE  0} {1 address_id  0} {1 in  0} {7 ( 0} {10 224, 0} {10 510) 0} {0   0} {5 ORDER BY  0} {1 last_name; 0}]
 SELECT * FROM film WHERE release_year = 2016 AND length != 1 ORDER BY title;
-[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 release_year  0} {7 = 0} {0   0} {10 2016  0} {6 AND  0} {4 LENGTH  0} {7 ! 0} {7 = 0} {0   0} {10 1  0} {5 ORDER BY  0} {1 title; 0}]
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 release_year  0} {7 = 0} {0   0} {10 2016  0} {6 AND  0} {4 LENGTH  0} {7 != 0} {0   0} {10 1  0} {5 ORDER BY  0} {1 title; 0}]
 SELECT title FROM film WHERE release_year = 1995;
 [{5 SELECT  0} {1 title  0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 release_year  0} {7 = 0} {0   0} {10 1995; 0}]
 SELECT title, replacement_cost FROM film WHERE language_id = 5 AND length = 70;
@@ -92,8 +92,8 @@ SELECT country_id, last_update FROM city NATURAL RIGHT JOIN country;
 [{5 SELECT  0} {1 country_id, 0} {0   0} {1 last_update  0} {5 FROM  0} {1 city  0} {1 NATURAL  0} {6 RIGHT JOIN  0} {1 country; 0}]
 SELECT a.country_id, a.last_update FROM city a STRAIGHT_JOIN country b ON a.country_id=b.country_id;
 [{5 SELECT  0} {1 a. 0} {1 country_id, 0} {0   0} {1 a. 0} {1 last_update  0} {5 FROM  0} {1 city  0} {1 a  0} {1 STRAIGHT_JOIN  0} {1 country  0} {1 b  0} {1 ON  0} {1 a. 0} {1 country_id= 0} {1 b. 0} {1 country_id; 0}]
-SELECT d.deptno,d.dname,d.loc FROM scott.dept d WHERE d.deptno IN  (SELECT e.deptno FROM scott.emp e);
-[{5 SELECT  0} {1 d. 0} {1 deptno, 0} {1 d. 0} {1 dname, 0} {1 d. 0} {1 loc  0} {5 FROM  0} {1 scott. 0} {1 dept  0} {1 d  0} {5 WHERE  0} {1 d. 0} {1 deptno  0} {1 IN  0} {0   0} {7 ( 0} {5 SELECT  0} {1 e. 0} {1 deptno  0} {5 FROM  0} {1 scott. 0} {1 emp  0} {1 e) 0} {7 ; 0}]
+SELECT a.address, a.postal_code FROM sakila.address a WHERE a.city_id IN  (SELECT c.city_id FROM sakila.city c);
+[{5 SELECT  0} {1 a. 0} {1 address, 0} {0   0} {1 a. 0} {1 postal_code  0} {5 FROM  0} {1 sakila. 0} {1 address  0} {1 a  0} {5 WHERE  0} {1 a. 0} {1 city_id  0} {1 IN  0} {0   0} {7 ( 0} {5 SELECT  0} {1 c. 0} {1 city_id  0} {5 FROM  0} {1 sakila. 0} {1 city  0} {1 c) 0} {7 ; 0}]
 SELECT visitor_id, url FROM (SELECT id FROM log WHERE ip="123.45.67.89" order by ts desc limit 50, 10) I JOIN log ON (I.id=log.id) JOIN url ON (url.id=log.url_id) order by TS desc;
 [{5 SELECT  0} {1 visitor_id, 0} {0   0} {1 url  0} {5 FROM  0} {7 ( 0} {5 SELECT  0} {1 id  0} {5 FROM  0} {4 LOG  0} {5 WHERE  0} {1 ip= 0} {2 "123.45.67.89" 0} {0   0} {5 ORDER BY  0} {1 ts  0} {1 desc  0} {5 LIMIT  0} {10 50, 0} {0   0} {10 10) 0} {0   0} {1 I  0} {6 JOIN  0} {4 LOG  0} {1 ON  0} {7 ( 0} {1 I. 0} {1 id= 0} {4 LOG. 0} {1 id) 0} {0   0} {6 JOIN  0} {1 url  0} {1 ON  0} {7 ( 0} {1 url. 0} {1 id= 0} {4 LOG. 0} {1 url_id) 0} {0   0} {5 ORDER BY  0} {1 TS  0} {1 desc; 0}]
 DELETE city, country FROM city INNER JOIN country using (country_id) WHERE city.city_id = 1;
@@ -160,7 +160,11 @@ alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
 [{5 ALTER TABLE  0} {1 inventory  0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 ; 0}]
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
 [{5 ALTER TABLE  0} {1 inventory  0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 , 0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 , 0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 ; 0}]
-SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
-[{5 SELECT	 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {7 , 0} {0   0} {4 COUNT( 0} {1 DISTINCT  0} {7 ( 0} {1 t. 0} {1 usr) 0} {7 ) 0} {0   0} {5 FROM  0} {1 usr_terminal  0} {1 t  0} {5 WHERE  0} {1 t. 0} {1 atm  0} {7 > 0} {0   0} {2 '2018-10-22 00:00:00' 0} {0   0} {6 AND  0} {1 t. 0} {1 agent  0} {1 LIKE  0} {2 '%Chrome%' 0} {0   0} {6 AND  0} {1 t. 0} {1 system  0} {7 = 0} {0   0} {2 'eip' 0} {0   0} {5 GROUP BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {0   0} {5 ORDER BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0}]
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
+[{5 SELECT	 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {7 , 0} {0   0} {4 COUNT( 0} {1 DISTINCT  0} {7 ( 0} {1 t. 0} {1 usr) 0} {7 ) 0} {0   0} {5 FROM  0} {1 usr_terminal  0} {1 t  0} {5 WHERE  0} {1 t. 0} {1 atm  0} {7 > 0} {0   0} {2 '2018-10-22 00:00:00' 0} {0   0} {6 AND  0} {1 t. 0} {1 agent  0} {1 LIKE  0} {2 '%Chrome%' 0} {0   0} {6 AND  0} {1 t. 0} {1 system  0} {7 = 0} {0   0} {2 'eip' 0} {0   0} {5 GROUP BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {0   0} {5 ORDER BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {7 ; 0}]
 create table hello.t (id int unsigned);
 [{1 create  0} {1 table  0} {1 hello. 0} {1 t  0} {7 ( 0} {1 id  0} {1 int  0} {1 unsigned) 0} {7 ; 0}]
+select * from tb where data >= '';
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 tb  0} {5 WHERE  0} {1 data  0} {7 >= 0} {0   0} {2 '' 0} {7 ; 0}]
+alter table tb alter column id drop default;
+[{5 ALTER TABLE  0} {1 tb  0} {1 alter  0} {1 column  0} {1 id  0} {5 DROP  0} {4 DEFAULT; 0}]

ast/testdata/TestTokenizer.golden

@@ -210,7 +210,7 @@
 }
 []ast.Token{
     {Type:57348, Val:"select", i:0},
-    {Type:57589, Val:"sql_calc_found_rows", i:0},
+    {Type:57590, Val:"sql_calc_found_rows", i:0},
     {Type:57396, Val:"col", i:0},
     {Type:57353, Val:"from", i:0},
     {Type:57396, Val:"tbl", i:0},

ast/testdata/TestVitessStmtNode2JSON.golden

+{
+  "Cache": "",
+  "Comments": null,
+  "Distinct": "",
+  "Hints": "",
+  "SelectExprs": [
+    {
+      "Expr": {
+        "Type": 1,
+        "Val": "MQ=="
+      },
+      "As": ""
+    }
+  ],
+  "From": [
+    {
+      "Expr": {
+        "Name": "dual",
+        "Qualifier": ""
+      },
+      "Partitions": null,
+      "As": "",
+      "Hints": null
+    }
+  ],
+  "Where": null,
+  "GroupBy": null,
+  "Having": null,
+  "OrderBy": null,
+  "Limit": null,
+  "Lock": ""
+}

ast/tidb.go

@@ -17,19 +17,27 @@
 package ast
 
 import (
+	"encoding/json"
+
 	"github.com/XiaoMi/soar/common"
 
 	"github.com/kr/pretty"
 	"github.com/pingcap/parser"
 	"github.com/pingcap/parser/ast"
-	// for pincap parser
+
+	// for pingcap parser
 	_ "github.com/pingcap/tidb/types/parser_driver"
 )
 
 // TiParse TiDB 语法解析
 func TiParse(sql, charset, collation string) ([]ast.StmtNode, error) {
 	p := parser.New()
-	return p.Parse(sql, charset, collation)
+	stmt, warn, err := p.Parse(sql, charset, collation)
+	// TODO: bypass warning info
+	for _, w := range warn {
+		common.Log.Warn(w.Error())
+	}
+	return stmt, err
 }
 
 // PrintPrettyStmtNode 打印TiParse语法树
@@ -43,20 +51,19 @@ func PrintPrettyStmtNode(sql, charset, collation string) {
 	}
 }
 
-// TiVisitor TODO:
-type TiVisitor struct {
-	EnterFunc func(node ast.Node) bool
-	LeaveFunc func(node ast.Node) bool
-}
-
-// Enter TODO:
-func (visitor *TiVisitor) Enter(n ast.Node) (node ast.Node, skip bool) {
-	skip = visitor.EnterFunc(n)
-	return
-}
-
-// Leave TODO:
-func (visitor *TiVisitor) Leave(n ast.Node) (node ast.Node, ok bool) {
-	ok = visitor.LeaveFunc(n)
-	return
+// StmtNode2JSON TiParse AST tree into json format
+func StmtNode2JSON(sql, charset, collation string) string {
+	var str string
+	tree, err := TiParse(sql, charset, collation)
+	if err != nil {
+		common.Log.Warning(err.Error())
+	} else {
+		b, err := json.MarshalIndent(tree, "", "  ")
+		if err != nil {
+			common.Log.Error(err.Error())
+		} else {
+			str = string(b)
+		}
+	}
+	return str
 }

ast/tidb_test.go

+/*
+ * Copyright 2018 Xiaomi, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ast
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/XiaoMi/soar/common"
+)
+
+func TestPrintPrettyStmtNode(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	sqls := []string{
+		`select 1`,
+	}
+	err := common.GoldenDiff(func() {
+		for _, sql := range sqls {
+			PrintPrettyStmtNode(sql, "", "")
+		}
+	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
+	}
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}
+
+func TestStmtNode2JSON(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	sqls := []string{
+		`select 1`,
+	}
+	err := common.GoldenDiff(func() {
+		for _, sql := range sqls {
+			fmt.Println(StmtNode2JSON(sql, "", ""))
+		}
+	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
+	}
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}

ast/token.go

@@ -48,7 +48,12 @@ var cacheHits int
 var cacheMisses int
 var tokenCache map[string]Token
 
-var tokenBoundaries = []string{",", ";", ":", ")", "(", ".", "=", "<", ">", "+", "-", "*", "/", "!", "^", "%", "|", "&", "#"}
+var tokenBoundaries = []string{
+	// multi character
+	"(>=)", "(<=)", "(!=)", "(<>)",
+	// single characters
+	",", ";", ":", "\\)", "\\(", "\\.", "=", "<", ">", "\\+", "-", "\\*", "/", "!", "\\^", "%", "\\|", "&", "#",
+}
 
 var tokenReserved = []string{
 	"ACCESSIBLE", "ACTION", "AGAINST", "AGGREGATE", "ALGORITHM", "ALL", "ALTER", "ANALYSE", "ANALYZE", "AS", "ASC",
@@ -124,9 +129,7 @@ var regFunctionString string
 
 func init() {
 	var regs []string
-	for _, reg := range tokenBoundaries {
-		regs = append(regs, regexp.QuoteMeta(reg))
-	}
+	regs = append(regs, tokenBoundaries...)
 	regBoundariesString = "(" + strings.Join(regs, "|") + ")"
 
 	regs = make([]string, 0)

ast/vitess.go

+/*
+ * Copyright 2018 Xiaomi, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ast
+
+import (
+	"encoding/json"
+
+	"github.com/XiaoMi/soar/common"
+
+	"github.com/kr/pretty"
+	"vitess.io/vitess/go/vt/sqlparser"
+)
+
+// PrintPrettyVitessStmtNode print vitess AST struct data
+func PrintPrettyVitessStmtNode(sql string) {
+	tree, err := sqlparser.Parse(sql)
+	if err != nil {
+		common.Log.Warning(err.Error())
+	} else {
+		_, err = pretty.Println(tree)
+		common.LogIfWarn(err, "")
+	}
+}
+
+// VitessStmtNode2JSON vitess AST tree into json format
+func VitessStmtNode2JSON(sql string) string {
+	var str string
+	tree, err := sqlparser.Parse(sql)
+	if err != nil {
+		common.Log.Warning(err.Error())
+	} else {
+		b, err := json.MarshalIndent(tree, "", "  ")
+		if err != nil {
+			common.Log.Error(err.Error())
+		} else {
+			str = string(b)
+		}
+	}
+	return str
+}

ast/vitess_test.go

+/*
+ * Copyright 2018 Xiaomi, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ast
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/XiaoMi/soar/common"
+)
+
+func TestPrintPrettyVitessStmtNode(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	sqls := []string{
+		`select 1`,
+	}
+	err := common.GoldenDiff(func() {
+		for _, sql := range sqls {
+			PrintPrettyVitessStmtNode(sql)
+		}
+	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
+	}
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}
+
+func TestVitessStmtNode2JSON(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	sqls := []string{
+		`select 1`,
+	}
+	err := common.GoldenDiff(func() {
+		for _, sql := range sqls {
+			fmt.Println(VitessStmtNode2JSON(sql))
+		}
+	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
+	}
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}

cmd/soar/soar.go

@@ -31,7 +31,6 @@ import (
 	"github.com/go-sql-driver/mysql"
 	"github.com/kr/pretty"
 	"github.com/percona/go-mysql/query"
-	"vitess.io/vitess/go/vt/sqlparser"
 )
 
 func main() {
@@ -143,20 +142,21 @@ func main() {
 			fmt.Println(ast.Compress(sql) + common.Config.Delimiter)
 			continue
 		case "ast":
-			// SQL 抽象语法树
-			var tree sqlparser.Statement
-			tree, err = sqlparser.Parse(sql)
-			if err != nil {
-				fmt.Println(err)
-			} else {
-				_, err = pretty.Println(tree)
-				common.LogIfWarn(err, "")
-			}
+			// print vitess AST data struct
+			ast.PrintPrettyVitessStmtNode(sql)
+			continue
+		case "ast-json":
+			// print vitess SQL AST into json format
+			fmt.Println(ast.VitessStmtNode2JSON(sql))
 			continue
 		case "tiast":
-			// TiDB SQL 抽象语法树
+			// print TiDB AST data struct
 			ast.PrintPrettyStmtNode(sql, "", "")
 			continue
+		case "tiast-json":
+			// print TiDB SQL AST into json format
+			fmt.Println(ast.StmtNode2JSON(sql, "", ""))
+			continue
 		case "tokenize":
 			// SQL 切词
 			_, err = pretty.Println(ast.Tokenize(sql))
@@ -185,16 +185,12 @@ func main() {
 		if syntaxErr != nil {
 			errContent := fmt.Sprintf("At SQL %d : %v", sqlCounter, syntaxErr)
 			common.Log.Warning(errContent)
-			if common.Config.OnlySyntaxCheck {
+			if common.Config.OnlySyntaxCheck || common.Config.ReportType == "rewrite" {
 				fmt.Println(errContent)
-			}
-			if !common.Config.DryRun {
 				os.Exit(1)
 			}
 			// tidb parser 语法检查给出的建议 ERR.000
-			if common.Config.TestDSN.Disable {
-				mysqlSuggest["ERR.000"] = advisor.RuleMySQLError("ERR.000", syntaxErr)
-			}
+			mysqlSuggest["ERR.000"] = advisor.RuleMySQLError("ERR.000", syntaxErr)
 		}
 		// 如果只想检查语法直接跳过后面的步骤
 		if common.Config.OnlySyntaxCheck {
@@ -255,6 +251,9 @@ func main() {
 							}
 						default:
 							// vEnv.VEnvBuild 阶段给出的 ERROR 是 ERR.001
+							if _, ok := mysqlSuggest["ERR.000"]; ok {
+								delete(mysqlSuggest, "ERR.000")
+							}
 							mysqlSuggest["ERR.001"] = advisor.RuleMySQLError("ERR.001", vEnv.Error)
 							common.Log.Error("BuildVirtualEnv DDL Execute Error : %v", vEnv.Error)
 						}
@@ -277,7 +276,7 @@ func main() {
 				explainInfo, err := rEnv.Explain(q.Query,
 					database.ExplainType[common.Config.ExplainType],
 					database.ExplainFormatType[common.Config.ExplainFormat])
-				if err != nil && strings.HasPrefix(vEnv.Database, "optimizer_") {
+				if err != nil {
 					// 线上环境执行失败才到测试环境 EXPLAIN，比如在用户提供建表语句及查询语句的场景
 					common.Log.Warn("rEnv.Explain Warn: %v", err)
 					explainInfo, err = vEnv.Explain(q.Query,
@@ -287,7 +286,6 @@ func main() {
 						// EXPLAIN 阶段给出的 ERROR 是 ERR.002
 						mysqlSuggest["ERR.002"] = advisor.RuleMySQLError("ERR.002", err)
 						common.Log.Error("vEnv.Explain Error: %v", err)
-						continue
 					}
 				}
 				// 分析 EXPLAIN 结果
@@ -426,4 +424,9 @@ func main() {
 		}
 		return
 	}
+
+	// syntax check verbose mode, add output for success!
+	if common.Config.OnlySyntaxCheck && common.Config.Verbose {
+		fmt.Println("Syntax check OK!")
+	}
 }

cmd/soar/soar_test.go

@@ -17,13 +17,28 @@
 package main
 
 import (
+	"flag"
 	"testing"
 
 	"github.com/XiaoMi/soar/common"
 )
 
-func init() {
-	common.Config.OnlineDSN.Schema = "sakila"
+var update = flag.Bool("update", false, "update .golden files")
+
+func TestMain(m *testing.M) {
+	// 初始化 init
+	common.BaseDir = common.DevPath
+	err := common.ParseConfig("")
+	common.LogIfError(err, "init ParseConfig")
+	common.Log.Debug("mysql_test init")
+	_ = update // check if var success init
+
+	// 分割线
+	flag.Parse()
+	m.Run()
+
+	// 环境清理
+	//
 }
 
 func Test_Main(_ *testing.T) {
@@ -45,3 +60,23 @@ func Test_Main_More(_ *testing.T) {
 		main()
 	}
 }
+
+func Test_Main_checkConfig(t *testing.T) {
+	if checkConfig() != 0 {
+		t.Error("checkConfig error")
+	}
+}
+
+func Test_Main_initQuery(t *testing.T) {
+	// direct query
+	query := initQuery("select 1")
+	if query != "select 1" {
+		t.Errorf("want 'select 1', got %s", query)
+	}
+
+	// read from file
+	initQuery(common.DevPath + "/README.md")
+
+	// TODO: read from stdin
+	// initQuery("")
+}

cmd/soar/tool.go

@@ -229,7 +229,9 @@ func shutdown(vEnv *env.VirtualEnv, rEnv *database.Connector) {
 	if common.Config.DropTestTemporary {
 		vEnv.CleanUp()
 	}
-	vEnv.Conn.Close()
-	rEnv.Conn.Close()
+	err := vEnv.Conn.Close()
+	common.LogIfWarn(err, "")
+	err = rEnv.Conn.Close()
+	common.LogIfWarn(err, "")
 	os.Exit(0)
 }

common/cases.go

@@ -136,8 +136,8 @@ func init() {
 		// SEMI JOIN
 		// 半连接： 当一张表在另一张表找到匹配的记录之后，半连接（semi-join）返回第一张表中的记录。
 		// 与条件连接相反，即使在右节点中找到几条匹配的记录，左节点的表也只会返回一条记录。
-		// 另外，右节点的表一条记录也不会返回。半连接通常使用IN  或 EXISTS 作为连接条件
-		"SELECT d.deptno,d.dname,d.loc FROM scott.dept d WHERE d.deptno IN  (SELECT e.deptno FROM scott.emp e);",
+		// 另外，右节点的表一条记录也不会返回。半连接通常使用 IN 或 EXISTS 作为连接条件
+		"SELECT a.address, a.postal_code FROM sakila.address a WHERE a.city_id IN  (SELECT c.city_id FROM sakila.city c);",
 
 		// Delayed Join
 		// https://www.percona.com/blog/2007/04/06/using-delayed-join-to-optimize-count-and-limit-queries/
@@ -196,8 +196,14 @@ func init() {
 		"alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);",
 
 		// https://github.com/XiaoMi/soar/issues/47
-		`SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')`,
+		`SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');`,
 		// https://github.com/XiaoMi/soar/issues/17
 		"create table hello.t (id int unsigned);",
+
+		// https://github.com/XiaoMi/soar/issues/146
+		"select * from tb where data >= '';",
+
+		// https://github.com/XiaoMi/soar/issues/163
+		"alter table tb alter column id drop default;",
 	}
 }

common/chardet.go

@@ -17,6 +17,7 @@
 package common
 
 import (
+	"github.com/kr/pretty"
 	"github.com/saintfish/chardet"
 )
 
@@ -39,6 +40,7 @@ func Chardet(buf []byte) string {
 	if err != nil {
 		return charset
 	}
+	Log.Debug("Chardet DetectAll Result: %s", pretty.Sprint(result))
 
 	// SOAR's main user speak Chinese, GB-18030, UTF-8 are higher suggested
 	for _, r := range result {

common/config.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"strings"
@@ -89,6 +90,7 @@ type Configuration struct {
 	MaxGroupByColsCount  int      `yaml:"max-group-by-cols-count"`   // 单条 SQL 中 GroupBy 包含列的最大数量
 	MaxDistinctCount     int      `yaml:"max-distinct-count"`        // 单条 SQL 中 Distinct 的最大数量
 	MaxIdxColsCount      int      `yaml:"max-index-cols-count"`      // 复合索引中包含列的最大数量
+	MaxTextColsCount     int      `yaml:"max-text-cols-count"`       // 表中含有的 text/blob 列的最大数量
 	MaxTotalRows         int64    `yaml:"max-total-rows"`            // 计算散粒度时，当数据行数大于 MaxTotalRows 即开启数据库保护模式，散粒度返回结果可信度下降
 	MaxQueryCost         int64    `yaml:"max-query-cost"`            // last_query_cost 超过该值时将给予警告
 	SpaghettiQueryLength int      `yaml:"spaghetti-query-length"`    // SQL最大长度警告，超过该长度会给警告
@@ -96,8 +98,9 @@ type Configuration struct {
 	MaxInCount           int      `yaml:"max-in-count"`              // IN()最大数量
 	MaxIdxBytesPerColumn int      `yaml:"max-index-bytes-percolumn"` // 索引中单列最大字节数，默认767
 	MaxIdxBytes          int      `yaml:"max-index-bytes"`           // 索引总长度限制，默认3072
-	TableAllowCharsets   []string `yaml:"table-allow-charsets"`      // TableName 允许使用的 DEFAULT CHARSET
-	TableAllowEngines    []string `yaml:"table-allow-engines"`       // TableName 允许使用的 Engine
+	AllowCharsets        []string `yaml:"allow-charsets"`            // 允许使用的 DEFAULT CHARSET
+	AllowCollates        []string `yaml:"allow-collates"`            // 允许使用的 COLLATE
+	AllowEngines         []string `yaml:"allow-engines"`             // 允许使用的存储引擎
 	MaxIdxCount          int      `yaml:"max-index-count"`           // 单张表允许最多索引数
 	MaxColCount          int      `yaml:"max-column-count"`          // 单张表允许最大列数
 	MaxValueCount        int      `yaml:"max-value-count"`           // INSERT/REPLACE 单次允许批量写入的行数
@@ -105,6 +108,8 @@ type Configuration struct {
 	UkPrefix             string   `yaml:"unique-key-prefix"`         // 唯一键建议使用的前缀
 	MaxSubqueryDepth     int      `yaml:"max-subquery-depth"`        // 子查询最大尝试
 	MaxVarcharLength     int      `yaml:"max-varchar-length"`        // varchar最大长度
+	ColumnNotAllowType   []string `yaml:"column-not-allow-type"`     // 字段不允许使用的数据类型
+	MinCardinality       float64  `yaml:"min-cardinality"`           // 添加索引散粒度阈值，范围 0~100
 
 	// ++++++++++++++EXPLAIN检查项+++++++++++++
 	ExplainSQLReportType   string   `yaml:"explain-sql-report-type"`  // EXPLAIN markdown 格式输出 SQL 样式，支持 sample, fingerprint, pretty 等
@@ -158,11 +163,13 @@ var Config = &Configuration{
 	Trace:                   false,
 	Explain:                 true,
 	Delimiter:               ";",
+	MinCardinality:          0,
 
 	MaxJoinTableCount:    5,
 	MaxGroupByColsCount:  5,
 	MaxDistinctCount:     5,
 	MaxIdxColsCount:      5,
+	MaxTextColsCount:     2,
 	MaxIdxBytesPerColumn: 767,
 	MaxIdxBytes:          3072,
 	MaxTotalRows:         9999999,
@@ -176,8 +183,9 @@ var Config = &Configuration{
 	ReportJavascript:     "",
 	ReportTitle:          "SQL优化分析报告",
 	BlackList:            "",
-	TableAllowCharsets:   []string{"utf8", "utf8mb4"},
-	TableAllowEngines:    []string{"innodb"},
+	AllowCharsets:        []string{"utf8", "utf8mb4"},
+	AllowCollates:        []string{},
+	AllowEngines:         []string{"innodb"},
 	MaxIdxCount:          10,
 	MaxColCount:          40,
 	MaxValueCount:        100,
@@ -186,6 +194,7 @@ var Config = &Configuration{
 	UkPrefix:             "uk_",
 	MaxSubqueryDepth:     5,
 	MaxVarcharLength:     1024,
+	ColumnNotAllowType:   []string{"boolean"},
 
 	MarkdownExtensions: 94,
 	MarkdownHTMLFlags:  0,
@@ -252,7 +261,8 @@ func parseDSN(odbc string, d *Dsn) *Dsn {
 	}
 
 	if d != nil {
-		addr = d.Addr
+		// 原来有个判断，后来判断条件被删除了就导致第一次addr无论如何都会被修改。所以这边先注释掉
+		// addr = d.Addr
 		user = d.User
 		password = d.Password
 		schema = d.Schema
@@ -265,60 +275,52 @@ func parseDSN(odbc string, d *Dsn) *Dsn {
 		return &Dsn{Disable: true}
 	}
 
-	// username:password@ip:port/database
-	l1 := strings.Split(odbc, "@")
-	if len(l1) < 2 {
-		if strings.HasPrefix(l1[0], ":") {
-			// ":port/database"
-			l2 := strings.Split(strings.TrimLeft(l1[0], ":"), "/")
-			if l2[0] == "" {
-				addr = strings.Split(addr, ":")[0] + ":3306"
-				if len(l2) > 1 {
-					schema = strings.Split(l2[1], "?")[0]
-				}
-			} else {
-				addr = strings.Split(addr, ":")[0] + ":" + l2[0]
-				if len(l2) > 1 {
-					schema = strings.Split(l2[1], "?")[0]
-				}
-			}
-		} else if strings.HasPrefix(l1[0], "/") {
-			// "/database"
-			l2 := strings.TrimLeft(l1[0], "/")
-			schema = l2
-		} else {
-			// ip:port/database
-			l2 := strings.Split(l1[0], "/")
-			if len(l2) == 2 {
-				addr = l2[0]
-				schema = strings.Split(l2[1], "?")[0]
-			} else {
-				addr = l2[0]
-			}
-		}
+	var userInfo, hostInfo, query string
+
+	// DSN 格式匹配
+	if res := regexp.MustCompile(`^(.*)@(.*?)/(.*?)($|\?)(.*)`).FindStringSubmatch(odbc); len(res) > 5 {
+		// userInfo@hostInfo/database
+		userInfo = res[1]
+		hostInfo = res[2]
+		schema = res[3]
+		query = res[5]
+	} else if res := regexp.MustCompile(`^(.*)/(.*?)($|\?)(.*)`).FindStringSubmatch(odbc); len(res) > 4 {
+		// hostInfo/database
+		hostInfo = res[1]
+		schema = res[2]
+		query = res[4]
+	} else if res := regexp.MustCompile(`^(.*)@(.*?)($|\?)(.*)`).FindStringSubmatch(odbc); len(res) > 4 {
+		// userInfo@hostInfo
+		userInfo = res[1]
+		hostInfo = res[2]
+		query = res[4]
 	} else {
-		// user:password
-		l2 := strings.Split(l1[0], ":")
-		if len(l2) == 2 {
-			user = l2[0]
-			password = l2[1]
-		} else {
-			user = l2[0]
-		}
-		// ip:port/database
-		l3 := strings.Split(l1[1], "/")
-		if len(l3) == 2 {
-			addr = l3[0]
-			schema = strings.Split(l3[1], "?")[0]
-		} else {
-			addr = l3[0]
-		}
+		// hostInfo
+		hostInfo = odbc
+	}
+
+	// 解析用户信息
+	if userInfo != "" {
+		user = strings.Split(userInfo, ":")[0]
+		// 防止密码中含有与用户名相同的字符, 所以用正则替换, 剩下的就是密码
+		password = strings.TrimLeft(regexp.MustCompile("^"+user).ReplaceAllString(userInfo, ""), ":")
+	}
+
+	// 解析主机信息
+	host := strings.Split(hostInfo, ":")[0]
+	port := strings.TrimLeft(strings.Replace(hostInfo, host, "", 1), ":")
+	if host == "" {
+		host = "127.0.0.1"
+	}
+	if port == "" {
+		port = "3306"
 	}
+	addr = host + ":" + port
 
-	// 其他flag参数，目前只支持charset :(
-	if len(strings.Split(odbc, "?")) > 1 {
-		flags := strings.Split(strings.Split(odbc, "?")[1], "&")
-		for _, f := range flags {
+	// 解析查询字符串
+	if query != "" {
+		params := strings.Split(query, "&")
+		for _, f := range params {
 			attr := strings.Split(f, "=")
 			if len(attr) > 1 {
 				arg := strings.TrimSpace(attr[0])
@@ -332,20 +334,6 @@ func parseDSN(odbc string, d *Dsn) *Dsn {
 		}
 	}
 
-	// 自动补端口
-	if !strings.Contains(addr, ":") {
-		addr = addr + ":3306"
-	} else {
-		if strings.HasSuffix(addr, ":") {
-			addr = addr + "3306"
-		}
-	}
-
-	// 默认走127.0.0.1
-	if strings.HasPrefix(addr, ":") {
-		addr = "127.0.0.1" + addr
-	}
-
 	// 默认用information_schema库
 	if schema == "" {
 		schema = "information_schema"
@@ -509,10 +497,11 @@ func readCmdFlags() error {
 	samplingStatisticTarget := flag.Int("sampling-statistic-target", Config.SamplingStatisticTarget, "SamplingStatisticTarget, 数据采样因子，对应 PostgreSQL 的 default_statistics_target")
 	samplingCondition := flag.String("sampling-condition", Config.SamplingCondition, "SamplingCondition, 数据采样条件，如： WHERE xxx LIMIT xxx")
 	delimiter := flag.String("delimiter", Config.Delimiter, "Delimiter, SQL分隔符")
+	minCardinality := flag.Float64("min-cardinality", Config.MinCardinality, "MinCardinality，索引列散粒度最低阈值，散粒度低于该值的列不添加索引，建议范围0.0 ~ 100.0")
 	// +++++++++++++++日志相关+++++++++++++++++
 	logLevel := flag.Int("log-level", Config.LogLevel, "LogLevel, 日志级别, [0:Emergency, 1:Alert, 2:Critical, 3:Error, 4:Warning, 5:Notice, 6:Informational, 7:Debug]")
 	logOutput := flag.String("log-output", Config.LogOutput, "LogOutput, 日志输出位置")
-	reportType := flag.String("report-type", Config.ReportType, "ReportType, 化建议输出格式，目前支持: json, text, markdown, html等")
+	reportType := flag.String("report-type", Config.ReportType, "ReportType, 优化建议输出格式，目前支持: json, text, markdown, html等")
 	reportCSS := flag.String("report-css", Config.ReportCSS, "ReportCSS, 当 ReportType 为 html 格式时使用的 css 风格，如不指定会提供一个默认风格。CSS可以是本地文件，也可以是一个URL")
 	reportJavascript := flag.String("report-javascript", Config.ReportJavascript, "ReportJavascript, 当 ReportType 为 html 格式时使用的javascript脚本，如不指定默认会加载SQL pretty 使用的 javascript。像CSS一样可以是本地文件，也可以是一个URL")
 	reportTitle := flag.String("report-title", Config.ReportTitle, "ReportTitle, 当 ReportType 为 html 格式时，HTML 的 title")
@@ -527,6 +516,7 @@ func readCmdFlags() error {
 	maxGroupByColsCount := flag.Int("max-group-by-cols-count", Config.MaxGroupByColsCount, "MaxGroupByColsCount, 单条 SQL 中 GroupBy 包含列的最大数量")
 	maxDistinctCount := flag.Int("max-distinct-count", Config.MaxDistinctCount, "MaxDistinctCount, 单条 SQL 中 Distinct 的最大数量")
 	maxIdxColsCount := flag.Int("max-index-cols-count", Config.MaxIdxColsCount, "MaxIdxColsCount, 复合索引中包含列的最大数量")
+	maxTextColsCount := flag.Int("max-text-cols-count", Config.MaxTextColsCount, "MaxTextColsCount, 表中含有的 text/blob 列的最大数量")
 	maxTotalRows := flag.Int64("max-total-rows", Config.MaxTotalRows, "MaxTotalRows, 计算散粒度时，当数据行数大于MaxTotalRows即开启数据库保护模式，不计算散粒度")
 	maxQueryCost := flag.Int64("max-query-cost", Config.MaxQueryCost, "MaxQueryCost, last_query_cost 超过该值时将给予警告")
 	spaghettiQueryLength := flag.Int("spaghetti-query-length", Config.SpaghettiQueryLength, "SpaghettiQueryLength, SQL最大长度警告，超过该长度会给警告")
@@ -534,8 +524,9 @@ func readCmdFlags() error {
 	maxInCount := flag.Int("max-in-count", Config.MaxInCount, "MaxInCount, IN()最大数量")
 	maxIdxBytesPerColumn := flag.Int("max-index-bytes-percolumn", Config.MaxIdxBytesPerColumn, "MaxIdxBytesPerColumn, 索引中单列最大字节数")
 	maxIdxBytes := flag.Int("max-index-bytes", Config.MaxIdxBytes, "MaxIdxBytes, 索引总长度限制")
-	tableAllowCharsets := flag.String("table-allow-charsets", strings.ToLower(strings.Join(Config.TableAllowCharsets, ",")), "TableAllowCharsets")
-	tableAllowEngines := flag.String("table-allow-engines", strings.ToLower(strings.Join(Config.TableAllowEngines, ",")), "TableAllowEngines")
+	allowCharsets := flag.String("allow-charsets", strings.ToLower(strings.Join(Config.AllowCharsets, ",")), "AllowCharsets")
+	allowCollates := flag.String("allow-collates", strings.ToLower(strings.Join(Config.AllowCollates, ",")), "AllowCollates")
+	allowEngines := flag.String("allow-engines", strings.ToLower(strings.Join(Config.AllowEngines, ",")), "AllowEngines")
 	maxIdxCount := flag.Int("max-index-count", Config.MaxIdxCount, "MaxIdxCount, 单表最大索引个数")
 	maxColCount := flag.Int("max-column-count", Config.MaxColCount, "MaxColCount, 单表允许的最大列数")
 	maxValueCount := flag.Int("max-value-count", Config.MaxValueCount, "MaxValueCount, INSERT/REPLACE 单次批量写入允许的行数")
@@ -543,6 +534,7 @@ func readCmdFlags() error {
 	ukPrefix := flag.String("unique-key-prefix", Config.UkPrefix, "UkPrefix")
 	maxSubqueryDepth := flag.Int("max-subquery-depth", Config.MaxSubqueryDepth, "MaxSubqueryDepth")
 	maxVarcharLength := flag.Int("max-varchar-length", Config.MaxVarcharLength, "MaxVarcharLength")
+	columnNotAllowType := flag.String("column-not-allow-type", strings.Join(Config.ColumnNotAllowType, ","), "ColumnNotAllowType")
 	// ++++++++++++++EXPLAIN检查项+++++++++++++
 	explainSQLReportType := flag.String("explain-sql-report-type", strings.ToLower(Config.ExplainSQLReportType), "ExplainSQLReportType [pretty, sample, fingerprint]")
 	explainType := flag.String("explain-type", strings.ToLower(Config.ExplainType), "ExplainType [extended, partitions, traditional]")
@@ -590,19 +582,13 @@ func readCmdFlags() error {
 	Config.SamplingCondition = *samplingCondition
 
 	Config.LogLevel = *logLevel
-	if strings.HasPrefix(*logOutput, "/") {
+
+	if filepath.IsAbs(*logOutput) || *logOutput == "" {
 		Config.LogOutput = *logOutput
 	} else {
-		if BaseDir == "" {
-			Config.LogOutput = *logOutput
-		} else {
-			if runtime.GOOS == "windows" {
-				Config.LogOutput = *logOutput
-			} else {
-				Config.LogOutput = BaseDir + "/" + *logOutput
-			}
-		}
+		Config.LogOutput = filepath.Join(BaseDir, *logOutput)
 	}
+
 	Config.ReportType = strings.ToLower(*reportType)
 	Config.ReportCSS = *reportCSS
 	Config.ReportJavascript = *reportJavascript
@@ -612,12 +598,14 @@ func readCmdFlags() error {
 	Config.IgnoreRules = strings.Split(*ignoreRules, ",")
 	Config.RewriteRules = strings.Split(*rewriteRules, ",")
 	*blackList = strings.TrimSpace(*blackList)
-	if strings.HasPrefix(*blackList, "/") || *blackList == "" {
+	Config.MinCardinality = *minCardinality
+
+	if filepath.IsAbs(*blackList) || *blackList == "" {
 		Config.BlackList = *blackList
 	} else {
-		pwd, _ := os.Getwd()
-		Config.BlackList = pwd + "/" + *blackList
+		Config.BlackList = filepath.Join(BaseDir, *blackList)
 	}
+
 	Config.MaxJoinTableCount = *maxJoinTableCount
 	Config.MaxGroupByColsCount = *maxGroupByColsCount
 	Config.MaxDistinctCount = *maxDistinctCount
@@ -628,10 +616,18 @@ func readCmdFlags() error {
 		Config.MaxIdxColsCount = 16
 	}
 
+	Config.MaxTextColsCount = *maxTextColsCount
 	Config.MaxIdxBytesPerColumn = *maxIdxBytesPerColumn
 	Config.MaxIdxBytes = *maxIdxBytes
-	Config.TableAllowCharsets = strings.Split(strings.ToLower(*tableAllowCharsets), ",")
-	Config.TableAllowEngines = strings.Split(strings.ToLower(*tableAllowEngines), ",")
+	if *allowCharsets != "" {
+		Config.AllowCharsets = strings.Split(strings.ToLower(*allowCharsets), ",")
+	}
+	if *allowCollates != "" {
+		Config.AllowCollates = strings.Split(strings.ToLower(*allowCollates), ",")
+	}
+	if *allowEngines != "" {
+		Config.AllowEngines = strings.Split(strings.ToLower(*allowEngines), ",")
+	}
 	Config.MaxIdxCount = *maxIdxCount
 	Config.MaxColCount = *maxColCount
 	Config.MaxValueCount = *maxValueCount
@@ -667,6 +663,9 @@ func readCmdFlags() error {
 	Config.DryRun = *dryrun
 	Config.MaxPrettySQLLength = *maxPrettySQLLength
 	Config.MaxVarcharLength = *maxVarcharLength
+	if *columnNotAllowType != "" {
+		Config.ColumnNotAllowType = strings.Split(strings.ToLower(*columnNotAllowType), ",")
+	}
 
 	PrintVersion = *printVersion
 	PrintConfig = *printConfig
@@ -684,8 +683,8 @@ func ParseConfig(configFile string) error {
 	if configFile == "" {
 		configs = []string{
 			"/etc/soar.yaml",
-			BaseDir + "/etc/soar.yaml",
-			BaseDir + "/soar.yaml",
+			filepath.Join(BaseDir, "etc", "soar.yaml"),
+			filepath.Join(BaseDir, "soar.yaml"),
 		}
 	} else {
 		configs = []string{

common/config_test.go

@@ -19,6 +19,7 @@ package common
 import (
 	"flag"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/kr/pretty"
@@ -26,8 +27,16 @@ import (
 
 var update = flag.Bool("update", false, "update .golden files")
 
-func init() {
+func TestMain(m *testing.M) {
+	// 初始化 init
 	BaseDir = DevPath
+
+	// 分割线
+	flag.Parse()
+	m.Run()
+
+	// 环境清理
+	//
 }
 
 func TestParseConfig(t *testing.T) {
@@ -41,7 +50,7 @@ func TestReadConfigFile(t *testing.T) {
 	if Config == nil {
 		Config = new(Configuration)
 	}
-	Config.readConfigFile(DevPath + "/soar.yaml")
+	Config.readConfigFile(filepath.Join(DevPath, "etc/soar.yaml"))
 }
 
 func TestParseDSN(t *testing.T) {
@@ -51,6 +60,8 @@ func TestParseDSN(t *testing.T) {
 		"user:password@hostname:3307",
 		"user:password@hostname:/database",
 		"user:password@:3307/database",
+		"user@hostname/dbname",
+		"user:pwd:pwd@pwd/pwd@hostname/dbname",
 		"user:password@",
 		"hostname:3307/database",
 		"@hostname:3307/database",
@@ -63,11 +74,14 @@ func TestParseDSN(t *testing.T) {
 		"/database",
 	}
 
-	GoldenDiff(func() {
+	err := GoldenDiff(func() {
 		for _, dsn := range dsns {
 			pretty.Println(parseDSN(dsn, nil))
 		}
 	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
+	}
 }
 
 func TestListReportTypes(t *testing.T) {
@@ -104,7 +118,14 @@ func TestArgConfig(t *testing.T) {
 }
 
 func TestPrintConfiguration(t *testing.T) {
-	Config.Verbose = true
-	PrintConfiguration()
-
+	Config.readConfigFile(filepath.Join(DevPath, "etc/soar.yaml"))
+	oldLogOutput := Config.LogOutput
+	Config.LogOutput = "soar.log"
+	err := GoldenDiff(func() {
+		PrintConfiguration()
+	}, t.Name(), update)
+	if err != nil {
+		t.Error(err)
+	}
+	Config.LogOutput = oldLogOutput
 }

common/logger.go

@@ -17,6 +17,7 @@
 package common
 
 import (
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"runtime"
@@ -40,7 +41,16 @@ func init() {
 func LoggerInit() {
 	Log.SetLevel(Config.LogLevel)
 	func() { _ = Log.DelLogger(logs.AdapterFile) }()
-	err := Log.SetLogger(logs.AdapterFile, fmt.Sprintf(`{"filename":"%s","level":7,"maxlines":0,"maxsize":0,"daily":false,"maxdays":0}`, Config.LogOutput))
+	logConfig := map[string]interface{}{
+		"filename": Config.LogOutput,
+		"level":    7,
+		"maxlines": 0,
+		"maxsize":  0,
+		"daily":    false,
+		"maxdays":  0,
+	}
+	logConfigJSON, _ := json.Marshal(logConfig)
+	err := Log.SetLogger(logs.AdapterFile, string(logConfigJSON))
 	if err != nil {
 		fmt.Println(err.Error())
 	}

common/testdata/TestParseDSN.golden

-&common.dsn{Addr:"", Schema:"", User:"", Password:"", Charset:"", Disable:true, Version:0}
-&common.dsn{Addr:"hostname:3307", Schema:"database", User:"user", Password:"password", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"hostname:3307", Schema:"information_schema", User:"user", Password:"password", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"hostname:3306", Schema:"database", User:"user", Password:"password", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"127.0.0.1:3307", Schema:"database", User:"user", Password:"password", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"127.0.0.1:3306", Schema:"information_schema", User:"user", Password:"password", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"hostname:3307", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"hostname:3307", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"hostname:3306", Schema:"information_schema", User:"", Password:"", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"hostname:3306", Schema:"information_schema", User:"", Password:"", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"127.0.0.1:3306", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"hostname:3307", Schema:"information_schema", User:"", Password:"", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"127.0.0.1:3307", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"127.0.0.1:3307", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Version:999}
-&common.dsn{Addr:"127.0.0.1:3306", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Version:999}
+(*common.Dsn)(nil)
+&common.Dsn{Net:"", Addr:"hostname:3307", Schema:"database", User:"user", Password:"password", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"hostname:3307", Schema:"information_schema", User:"user", Password:"password", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"hostname:3306", Schema:"database", User:"user", Password:"password", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"127.0.0.1:3307", Schema:"database", User:"user", Password:"password", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"hostname:3306", Schema:"dbname", User:"user", Password:"", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"hostname:3306", Schema:"dbname", User:"user", Password:"pwd:pwd@pwd/pwd", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"127.0.0.1:3306", Schema:"information_schema", User:"user", Password:"password", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"hostname:3307", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"hostname:3307", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"hostname:3306", Schema:"information_schema", User:"", Password:"", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"hostname:3306", Schema:"information_schema", User:"", Password:"", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"127.0.0.1:3306", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"hostname:3307", Schema:"information_schema", User:"", Password:"", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"127.0.0.1:3307", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"127.0.0.1:3307", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}
+&common.Dsn{Net:"", Addr:"127.0.0.1:3306", Schema:"database", User:"", Password:"", Charset:"utf8mb4", Disable:false, Timeout:0, ReadTimeout:0, WriteTimeout:0, Version:999}

common/testdata/TestPrintConfiguration.golden

 online-dsn:
-  addr: ""
-  schema: information_schema
-  user: ""
-  password: ""
+  net: tcp
+  addr: 127.0.0.1:3306
+  schema: sakila
+  user: root
+  password: '********'
   charset: utf8mb4
-  disable: true
+  disable: false
+  timeout: 0
+  read-timeout: 0
+  write-timeout: 0
 test-dsn:
-  addr: ""
-  schema: information_schema
-  user: ""
-  password: ""
+  net: tcp
+  addr: 127.0.0.1:3306
+  schema: sakila
+  user: root
+  password: '********'
   charset: utf8mb4
-  disable: true
-allow-online-as-test: false
+  disable: false
+  timeout: 0
+  read-timeout: 0
+  write-timeout: 0
+allow-online-as-test: true
 drop-test-temporary: true
 cleanup-test-database: false
 only-syntax-check: false
 sampling-statistic-target: 100
-sampling: false
+sampling: true
+sampling-condition: ""
 profiling: false
 trace: false
 explain: true
-conn-time-out: 3
-query-time-out: 30
 delimiter: ;
-log-level: 3
-log-output: /dev/stderr
+log-level: 7
+log-output: soar.log
 report-type: markdown
 report-css: ""
 report-javascript: ""
@@ -48,6 +55,7 @@ max-join-table-count: 5
 max-group-by-cols-count: 5
 max-distinct-count: 5
 max-index-cols-count: 5
+max-text-cols-count: 2
 max-total-rows: 9999999
 max-query-cost: 9999
 spaghetti-query-length: 2048
@@ -55,17 +63,22 @@ allow-drop-index: false
 max-in-count: 10
 max-index-bytes-percolumn: 767
 max-index-bytes: 3072
-table-allow-charsets:
+allow-charsets:
 - utf8
 - utf8mb4
-table-allow-engines:
+allow-collates: []
+allow-engines:
 - innodb
 max-index-count: 10
 max-column-count: 40
+max-value-count: 100
 index-prefix: idx_
 unique-key-prefix: uk_
 max-subquery-depth: 5
 max-varchar-length: 1024
+column-not-allow-type:
+- boolean
+min-cardinality: 0
 explain-sql-report-type: pretty
 explain-type: extended
 explain-format: traditional
@@ -89,6 +102,6 @@ list-heuristic-rules: false
 list-rewrite-rules: false
 list-test-sqls: false
 list-report-types: false
-verbose: true
+verbose: false
 dry-run: true
 max-pretty-sql-length: 1024

database/explain.go

@@ -939,6 +939,9 @@ func ParseExplainResult(res QueryResult, formatType int) (exp *ExplainInfo, err
 		if res.Rows.Next() {
 			var explainString string
 			err = res.Rows.Scan(&explainString)
+			if err != nil {
+				common.Log.Debug(err.Error())
+			}
 			exp.ExplainJSON, err = parseJSONExplainText(explainString)
 		}
 		res.Rows.Close()
@@ -979,7 +982,10 @@ func ParseExplainResult(res QueryResult, formatType int) (exp *ExplainInfo, err
 	var explainRows []*ExplainRow
 
 	for res.Rows.Next() {
-		res.Rows.Scan(explainFields...)
+		err := res.Rows.Scan(explainFields...)
+		if err != nil {
+			common.Log.Debug(err.Error())
+		}
 		expRow.PossibleKeys = strings.Split(possibleKeys, ",")
 
 		// MySQL bug: https://bugs.mysql.com/bug.php?id=34124

database/explain_test.go

@@ -2332,7 +2332,7 @@ possible_keys: idx_fk_country_id,idx_country_id_city,idx_all,idx_other
 }
 
 func TestExplain(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	// TraditionalFormatExplain
 	for idx, sql := range sqls {
 		exp, err := connTest.Explain(sql, TraditionalExplainType, TraditionalFormatExplain)
@@ -2355,7 +2355,7 @@ func TestExplain(t *testing.T) {
 }
 
 func TestParseExplainText(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	for _, content := range exp {
 		pretty.Println(RemoveSQLComments(content))
 		pretty.Println(ParseExplainText(content))
@@ -2371,7 +2371,7 @@ func TestParseExplainText(t *testing.T) {
 }
 
 func TestFindTablesInJson(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	idx := 9
 	for _, j := range exp[idx : idx+1] {
 		pretty.Println(j)
@@ -2382,7 +2382,7 @@ func TestFindTablesInJson(t *testing.T) {
 }
 
 func TestFormatJsonIntoTraditional(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	idx := 11
 	for _, j := range exp[idx : idx+1] {
 		pretty.Println(j)
@@ -2392,7 +2392,7 @@ func TestFormatJsonIntoTraditional(t *testing.T) {
 }
 
 func TestPrintMarkdownExplainTable(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	expInfo, err := connTest.Explain("select 1", TraditionalExplainType, TraditionalFormatExplain)
 	if err != nil {
 		t.Error(err)
@@ -2408,7 +2408,7 @@ func TestPrintMarkdownExplainTable(t *testing.T) {
 }
 
 func TestExplainInfoTranslator(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	expInfo, err := connTest.Explain("select 1", TraditionalExplainType, TraditionalFormatExplain)
 	if err != nil {
 		t.Error(err)
@@ -2423,7 +2423,7 @@ func TestExplainInfoTranslator(t *testing.T) {
 }
 
 func TestMySQLExplainWarnings(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	expInfo, err := connTest.Explain("select 1", TraditionalExplainType, TraditionalFormatExplain)
 	if err != nil {
 		t.Error(err)
@@ -2438,7 +2438,7 @@ func TestMySQLExplainWarnings(t *testing.T) {
 }
 
 func TestMySQLExplainQueryCost(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	expInfo, err := connTest.Explain("select 1", TraditionalExplainType, TraditionalFormatExplain)
 	if err != nil {
 		t.Error(err)
@@ -2453,7 +2453,7 @@ func TestMySQLExplainQueryCost(t *testing.T) {
 }
 
 func TestSupportExplainWrite(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	_, err := connTest.supportExplainWrite()
 	if err != nil {
 		t.Error(err)
@@ -2462,7 +2462,7 @@ func TestSupportExplainWrite(t *testing.T) {
 }
 
 func TestExplainAbleSQL(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	for _, sql := range sqls {
 		if _, err := connTest.explainAbleSQL(sql); err != nil {
 			t.Errorf("SQL: %s, not explain able", sql)

database/mysql.go

@@ -91,9 +91,16 @@ func (db *Connector) Query(sql string, params ...interface{}) (QueryResult, erro
 			db.Addr, db.Database, fmt.Sprintf(sql, params...))
 	}
 
+	if db.Database == "" {
+		db.Database = "information_schema"
+	}
+
 	common.Log.Debug("Execute SQL with DSN(%s/%s) : %s", db.Addr, db.Database, fmt.Sprintf(sql, params...))
 	_, err = db.Conn.Exec("USE " + db.Database)
-	common.LogIfError(err, "")
+	if err != nil {
+		common.Log.Error(err.Error())
+		return res, err
+	}
 	res.Rows, res.Error = db.Conn.Query(sql, params...)
 
 	if common.Config.ShowWarnings {
@@ -175,6 +182,7 @@ func (db *Connector) ColumnCardinality(tb, col string) float64 {
 	// 获取该表上的已有的索引
 
 	// show table status 获取总行数（近似）
+	common.Log.Debug("ColumnCardinality, ShowTableStatus check `%s` status Rows", tb)
 	tbStatus, err := db.ShowTableStatus(tb)
 	if err != nil {
 		common.Log.Warn("(db *Connector) ColumnCardinality() ShowTableStatus Error: %v", err)
@@ -227,6 +235,7 @@ func (db *Connector) ColumnCardinality(tb, col string) float64 {
 
 // IsView 判断表是否是视图
 func (db *Connector) IsView(tbName string) bool {
+	common.Log.Debug("IsView, ShowTableStatus check `%s` is view", tbName)
 	tbStatus, err := db.ShowTableStatus(tbName)
 	if err != nil {
 		common.Log.Error("(db *Connector) IsView Error: %v:", err)
@@ -289,7 +298,7 @@ func (db *Connector) dangerousQuery(query string) bool {
 	return false
 }
 
-// Sandard MySQL datetime format
+// TimeFormat standard MySQL datetime format
 const TimeFormat = "2006-01-02 15:04:05.000000000"
 
 // TimeString returns t as string in MySQL format Converts time.Time zero to MySQL zero.

database/mysql_test.go

@@ -30,7 +30,8 @@ import (
 var connTest *Connector
 var update = flag.Bool("update", false, "update .golden files")
 
-func init() {
+func TestMain(m *testing.M) {
+	// 初始化 init
 	common.BaseDir = common.DevPath
 	err := common.ParseConfig("")
 	common.LogIfError(err, "init ParseConfig")
@@ -45,6 +46,13 @@ func init() {
 		common.Log.Critical("Test env Error: %v", err)
 		os.Exit(0)
 	}
+
+	// 分割线
+	flag.Parse()
+	m.Run()
+
+	// 环境清理
+	//
 }
 
 func TestQuery(t *testing.T) {

database/profiling.go

@@ -68,7 +68,12 @@ func (db *Connector) Profiling(sql string, params ...interface{}) ([]ProfilingRo
 	if err != nil {
 		return rows, err
 	}
-	defer trx.Rollback()
+	defer func() {
+		trxErr := trx.Rollback()
+		if trxErr != nil {
+			common.Log.Debug(trxErr.Error())
+		}
+	}()
 
 	// 开启 Profiling
 	_, err = trx.Query("set @@profiling=1")
@@ -86,7 +91,10 @@ func (db *Connector) Profiling(sql string, params ...interface{}) ([]ProfilingRo
 	// 返回 Profiling 结果
 	res, err := trx.Query("show profile")
 	if err != nil {
-		trx.Rollback()
+		trxErr := trx.Rollback()
+		if trxErr != nil {
+			common.Log.Debug(trxErr.Error())
+		}
 		return rows, err
 	}
 	var profileRow ProfilingRow

database/sampling.go

@@ -46,10 +46,10 @@ import (
  */
 
 // SamplingData 将数据从 onlineConn 拉取到 db 中
-func (db *Connector) SamplingData(onlineConn *Connector, database string, tables ...string) error {
+func (db *Connector) SamplingData(onlineConn *Connector, tables ...string) error {
 	var err error
-	if database == db.Database {
-		return fmt.Errorf("SamplingData the same database, From: %s/%s, To: %s/%s", onlineConn.Addr, database, db.Addr, db.Database)
+	if onlineConn.Database == db.Database {
+		return fmt.Errorf("SamplingData the same database, From: %s/%s, To: %s/%s", onlineConn.Addr, onlineConn.Database, db.Addr, db.Database)
 	}
 
 	// 计算需要泵取的数据量
@@ -82,21 +82,21 @@ func (db *Connector) SamplingData(onlineConn *Connector, database string, tables
 
 			factor := float64(wantRowsCount) / float64(tableRows)
 			common.Log.Debug("SamplingData, tableRows: %d, wantRowsCount: %d, factor: %f", tableRows, wantRowsCount, factor)
-			where = fmt.Sprintf("WHERE RAND() <= %f LIMIT %d", factor, wantRowsCount)
+			where = fmt.Sprintf("where RAND() <= %f LIMIT %d", factor, wantRowsCount)
 			if factor >= 1 {
 				where = ""
 			}
 		} else {
 			where = common.Config.SamplingCondition
 		}
-		err = db.startSampling(onlineConn.Conn, database, table, where)
+		err = db.startSampling(onlineConn.Conn, onlineConn.Database, table, where)
 	}
 	return err
 }
 
 // startSampling sampling data from OnlineDSN to TestDSN
 func (db *Connector) startSampling(onlineConn *sql.DB, database, table string, where string) error {
-	samplingQuery := fmt.Sprintf("SELECT * FROM `%s`.`%s` %s", database, table, where)
+	samplingQuery := fmt.Sprintf("select * from `%s`.`%s` %s", database, table, where)
 	common.Log.Debug("startSampling with Query: %s", samplingQuery)
 	res, err := onlineConn.Query(samplingQuery)
 	if err != nil {
@@ -125,7 +125,10 @@ func (db *Connector) startSampling(onlineConn *sql.DB, database, table string, w
 	columnsStr := "`" + strings.Join(columns, "`,`") + "`"
 	for res.Next() {
 		var values []string
-		res.Scan(tableFields...)
+		err = res.Scan(tableFields...)
+		if err != nil {
+			common.Log.Debug(err.Error())
+		}
 		for i, val := range row {
 			if val == nil {
 				values = append(values, "NULL")
@@ -147,7 +150,6 @@ func (db *Connector) startSampling(onlineConn *sql.DB, database, table string, w
 			if err != nil {
 				break
 			}
-			values = make([]string, 0)
 			valuesStr = make([]string, 0)
 			valuesCount = 0
 		}
@@ -159,7 +161,7 @@ func (db *Connector) startSampling(onlineConn *sql.DB, database, table string, w
 // 将泵取的数据转换成 insert 语句并在 testConn 数据库中执行
 func (db *Connector) doSampling(table, colDef, values string) error {
 	// db.Database is hashed database name
-	query := fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES %s;", db.Database, table, colDef, values)
+	query := fmt.Sprintf("insert into `%s`.`%s` (%s) values %s;", db.Database, table, colDef, values)
 	res, err := db.Query(query)
 	if res.Rows != nil {
 		res.Rows.Close()

database/show.go

@@ -154,7 +154,10 @@ func (db *Connector) ShowTableStatus(tableName string) (*TableStatInfo, error) {
 	}
 	// 获取值
 	for res.Rows.Next() {
-		res.Rows.Scan(statusFields...)
+		err := res.Rows.Scan(statusFields...)
+		if err != nil {
+			common.Log.Debug(err.Error())
+		}
 		tbStatus.Rows = append(tbStatus.Rows, ts)
 	}
 	res.Rows.Close()
@@ -243,7 +246,10 @@ func (db *Connector) ShowIndex(tableName string) (*TableIndexInfo, error) {
 	}
 	// 获取值
 	for res.Rows.Next() {
-		res.Rows.Scan(indexFields...)
+		err := res.Rows.Scan(indexFields...)
+		if err != nil {
+			common.Log.Debug(err.Error())
+		}
 		tbIndex.Rows = append(tbIndex.Rows, ti)
 	}
 	res.Rows.Close()
@@ -374,7 +380,10 @@ func (db *Connector) ShowColumns(tableName string) (*TableDesc, error) {
 	}
 	// 获取值
 	for res.Rows.Next() {
-		res.Rows.Scan(columnFields...)
+		err := res.Rows.Scan(columnFields...)
+		if err != nil {
+			common.Log.Debug(err.Error())
+		}
 		tbDesc.DescValues = append(tbDesc.DescValues, tc)
 	}
 	res.Rows.Close()
@@ -399,7 +408,7 @@ func (db *Connector) showCreate(createType, name string) (string, error) {
 	// SHOW CREATE TABLE tbl_name
 	// SHOW CREATE TRIGGER trigger_name
 	// SHOW CREATE VIEW view_name
-	res, err := db.Query(fmt.Sprintf("show create %s `%s`", createType, name))
+	res, err := db.Query(fmt.Sprintf("SHOW CREATE %s `%s`", createType, name))
 	if err != nil {
 		return "", err
 	}
@@ -423,14 +432,17 @@ func (db *Connector) showCreate(createType, name string) (string, error) {
 		if _, ok := fields[col]; ok {
 			createFields = append(createFields, fields[col])
 		} else {
-			common.Log.Debug("showCreate by pass column %s", col)
+			common.Log.Debug("showCreate Type: %s, Name: %s, by pass column `%s`", createType, name, col)
 			createFields = append(createFields, &colByPass)
 		}
 	}
 
 	// 获取 CREATE 语句
 	for res.Rows.Next() {
-		res.Rows.Scan(createFields...)
+		err := res.Rows.Scan(createFields...)
+		if err != nil {
+			common.Log.Debug(err.Error())
+		}
 	}
 	res.Rows.Close()
 	return create, err
@@ -456,7 +468,7 @@ func (db *Connector) ShowCreateTable(tableName string) (string, error) {
 		}
 	}()
 
-	ddl, err := db.showCreate("table", tableName)
+	ddl, err := db.showCreate("TABLE", tableName)
 
 	// 去除外键关联条件
 	lines := strings.Split(ddl, "\n")

database/trace.go

@@ -77,7 +77,12 @@ func (db *Connector) Trace(sql string, params ...interface{}) ([]TraceRow, error
 	if err != nil {
 		return rows, err
 	}
-	defer trx.Rollback()
+	defer func() {
+		trxErr := trx.Rollback()
+		if trxErr != nil {
+			common.Log.Debug(trxErr.Error())
+		}
+	}()
 	_, err = trx.Query("SET SESSION OPTIMIZER_TRACE='enabled=on'")
 	common.LogIfError(err, "")
 
@@ -93,7 +98,10 @@ func (db *Connector) Trace(sql string, params ...interface{}) ([]TraceRow, error
 	// 返回Trace结果
 	res, err := trx.Query("SELECT * FROM information_schema.OPTIMIZER_TRACE")
 	if err != nil {
-		trx.Rollback()
+		trxErr := trx.Rollback()
+		if trxErr != nil {
+			common.Log.Debug(trxErr.Error())
+		}
 		return rows, err
 	}
 	for res.Next() {

deps.sh

 #!/bin/bash
 
-NEEDED_COMMANDS="mysql docker git go govendor retool"
+NEEDED_COMMANDS="docker git go govendor retool"
 
 for cmd in ${NEEDED_COMMANDS} ; do
     if ! command -v "${cmd}" &> /dev/null ; then
@@ -10,3 +10,18 @@ for cmd in ${NEEDED_COMMANDS} ; do
         echo "${cmd} found"
     fi
 done
+
+# Docker
+## https://www.docker.com
+
+# Git
+## https://git-scm.com/
+
+# Go
+## https://golang.org/
+
+# Govendor
+## go get github.com/kardianos/govendor
+
+# retool
+## go get github.com/twitchtv/retool

doc/FAQ.md

@@ -48,6 +48,16 @@ bash: ./soar.linux-amd64: cannot execute binary file
 
 请注意您操作系统类型，`soar.linux-amd64` 为 Linux 系统使用的二进制文件，`soar.darwin-amd64` 为苹果系统使用的二进制文件，`soar.windows-amd64` 是微软用户使用的二进制文件。下载文件后 Linux 和苹果用户需要为文件添加可执行权限 `chmod a+x filename`。
 
+## 命令无法找到
+
+```bash
+bash: soar: command not found
+```
+
+直接执行 `soar` 命令提示命令无法找到，请先将 soar 文件添加可执行权限 `chmod a+x soar` 然后将可以将 soar 所在路径加到[PATH](https://linuxconfig.org/linux-path-environment-variable)中，也可以将 soar 移动到已有 PATH 中。
+
+当然在 Linux 环境下，在 soar 二进制文件所在路径运行 `./soar` 也同样可以解决您的问题。
+
 ## 提示语法错误
 
 * 请检查SQL语句中是否出现了不配对的引号,如 `, ", '

doc/example/main_test.sh

 #!/bin/bash
 
-
+GOPATH=$(go env GOPATH)
 PROJECT_PATH=${GOPATH}/src/github.com/XiaoMi/soar/
 
 if [ "$1x" == "-updatex" ]; then
-	cd "${PROJECT_PATH}" && ./bin/soar -list-test-sqls | ./bin/soar -config=../etc/soar.yaml > ./doc/example/main_test.md
+  cd "${PROJECT_PATH}" && ./bin/soar -list-test-sqls | ./bin/soar -config=../etc/soar.yaml > ./doc/example/main_test.md
+  if [ ! $? -eq 0 ]; then
+    exit 1
+  fi
 else
-	cd "${PROJECT_PATH}" && ./bin/soar -list-test-sqls | ./bin/soar -config=../etc/soar.yaml > ./doc/example/main_test.log
-    # optimizer_XXX 库名，散粒度，以及索引先后顺序每次可能会不一致
-    DIFF_LINES=$(cat ./doc/example/main_test.log ./doc/example/main_test.md | grep -v "optimizer\|散粒度" | sort | uniq -u | wc -l)
-    if [ "${DIFF_LINES}" -gt 0 ]; then
-    	git diff ./doc/example/main_test.log ./doc/example/main_test.md
-    fi
+  cd "${PROJECT_PATH}" && ./bin/soar -list-test-sqls | ./bin/soar -config=../etc/soar.yaml > ./doc/example/main_test.log
+  if [ ! $? -eq 0 ]; then
+    exit 1
+  fi
+  # optimizer_XXX 库名，散粒度，以及索引先后顺序每次可能会不一致
+  DIFF_LINES=$(cat ./doc/example/main_test.log ./doc/example/main_test.md | grep -v "optimizer\|散粒度" | sort | uniq -u | wc -l)
+  if [ "${DIFF_LINES}" -gt 0 ]; then
+    git diff ./doc/example/main_test.log ./doc/example/main_test.md
+  fi
 fi
-
-

doc/heuristic.md

@@ -409,6 +409,16 @@ CREATE TABLE tbl (col int) ENGINE=InnoDB;
 * **Content**:表中包含有太多的列
 * **Case**:
 
+```sql
+CREATE TABLE tbl ( cols ....);
+```
+## 表中包含有太多的 text/blob 列
+
+* **Item**:COL.007
+* **Severity**:L3
+* **Content**:表中包含超过2个的 text/blob 列
+* **Case**:
+
 ```sql
 CREATE TABLE tbl ( cols ....);
 ```
@@ -452,15 +462,15 @@ create table tab1(status ENUM('new','in progress','fixed'))
 ```sql
 select c1,c2,c3 from tbl where c4 is null or c4 <> 1
 ```
-## BLOB 和 TEXT 类型的字段不可设置为 NULL
+## BLOB 和 TEXT 类型的字段不建议设置为 NOT NULL
 
 * **Item**:COL.012
 * **Severity**:L5
-* **Content**:BLOB 和 TEXT 类型的字段不可设置为 NULL
+* **Content**:BLOB 和 TEXT 类型的字段无法指定非 NULL 的默认值，如果添加了 NOT NULL 限制，写入数据时又未对该字段指定值可能导致写入失败。
 * **Case**:
 
 ```sql
-CREATE TABLE `tbl` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `c` longblob, PRIMARY KEY (`id`));
+CREATE TABLE `tb`(`c` longblob NOT NULL);
 ```
 ## TIMESTAMP 类型未设置默认值
 
@@ -482,15 +492,15 @@ CREATE TABLE tbl( `id` bigint not null, `create_time` timestamp);
 ```sql
 CREATE TABLE `tb2` ( `id` int(11) DEFAULT NULL, `col` char(10) CHARACTER SET utf8 DEFAULT NULL)
 ```
-## BLOB 类型的字段不可指定默认值
+## TEXT 和 BLOB 类型的字段不可指定非 NULL 的默认值
 
 * **Item**:COL.015
 * **Severity**:L4
-* **Content**:BLOB 类型的字段不可指定默认值
+* **Content**:MySQL 数据库中 TEXT 和 BLOB 类型的字段不可指定非 NULL 的默认值。TEXT最大长度为2^16-1个字符，MEDIUMTEXT最大长度为2^32-1个字符，LONGTEXT最大长度为2^64-1个字符。
 * **Case**:
 
 ```sql
-CREATE TABLE `tbl` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `c` blob NOT NULL DEFAULT '', PRIMARY KEY (`id`));
+CREATE TABLE `tbl` (`c` blob DEFAULT NULL);
 ```
 ## 整型定义建议采用 INT(10) 或 BIGINT(20)
 
@@ -512,6 +522,26 @@ CREATE TABLE tab (a INT(1));
 ```sql
 CREATE TABLE tab (a varchar(3500));
 ```
+## 建表语句中使用了不推荐的字段类型
+
+* **Item**:COL.018
+* **Severity**:L1
+* **Content**:以下字段类型不被推荐使用：boolean
+* **Case**:
+
+```sql
+CREATE TABLE tab (a BOOLEAN);
+```
+## 不建议使用精度在秒级以下的时间数据类型
+
+* **Item**:COL.019
+* **Severity**:L1
+* **Content**:使用高精度的时间数据类型带来的存储空间消耗相对较大；MySQL 在5.6.4以上才可以支持精确到微秒的时间数据类型，使用时需要考虑版本兼容问题。
+* **Case**:
+
+```sql
+CREATE TABLE t1 (t TIME(3), dt DATETIME(6));
+```
 ## 消除不必要的 DISTINCT 条件
 
 * **Item**:DIS.001
@@ -1002,6 +1032,16 @@ select * from tbl where 1 = 1;
 ```sql
 LOAD DATA INFILE 'data.txt' INTO TABLE db2.my_table;
 ```
+## 不建议使用连续判断
+
+* **Item**:RES.009
+* **Severity**:L2
+* **Content**:类似这样的 SELECT \* FROM tbl WHERE col = col = 'abc' 语句可能是书写错误，您可能想表达的含义是 col = 'abc'。如果确实是业务需求建议修改为 col = col and col = 'abc'。
+* **Case**:
+
+```sql
+SELECT * FROM tbl WHERE col = col = 'abc'
+```
 ## 请谨慎使用TRUNCATE操作
 
 * **Item**:SEC.001
@@ -1176,7 +1216,7 @@ CREATE TABLE tbl (a int) AUTO_INCREMENT = 10;
 
 * **Item**:TBL.005
 * **Severity**:L4
-* **Content**:表字符集只允许设置为utf8,utf8mb4
+* **Content**:表字符集只允许设置为'utf8,utf8mb4'
 * **Case**:
 
 ```sql
@@ -1202,3 +1242,13 @@ create view v_today (today) AS SELECT CURRENT_DATE;
 ```sql
 CREATE TEMPORARY TABLE `work` (`time` time DEFAULT NULL) ENGINE=InnoDB;
 ```
+## 请使用推荐的COLLATE
+
+* **Item**:TBL.008
+* **Severity**:L4
+* **Content**:COLLATE 只允许设置为''
+* **Case**:
+
+```sql
+CREATE TABLE tbl (a int) DEFAULT COLLATE = latin1_bin;
+```

env/env.go

@@ -34,10 +34,10 @@ import (
 type VirtualEnv struct {
 	*database.Connector
 
-	// 保存DB测试环境映射关系，防止vEnv环境冲突。
+	// 保存 DB 测试环境映射关系，防止 vEnv 环境冲突。
 	DBRef   map[string]string // db -> optimizer_xxx
-	hash2Db map[string]string // optimizer_xxx -> db
-	// 保存Table创建关系，防止重复创建表
+	Hash2DB map[string]string // optimizer_xxx -> db
+	// 保存 Table 创建关系，防止重复创建表
 	TableMap map[string]map[string]string
 	// 错误
 	Error error
@@ -48,7 +48,7 @@ func NewVirtualEnv(vEnv *database.Connector) *VirtualEnv {
 	return &VirtualEnv{
 		Connector: vEnv,
 		DBRef:     make(map[string]string),
-		hash2Db:   make(map[string]string),
+		Hash2DB:   make(map[string]string),
 		TableMap:  make(map[string]map[string]string),
 	}
 }
@@ -86,7 +86,7 @@ func BuildEnv() (*VirtualEnv, *database.Connector) {
 	common.Config.OnlineDSN.Version = rEnvVersion
 	if err != nil {
 		common.Log.Warn("BuildEnv OnlineDSN: %s:********@%s/%s not available , Error: %s",
-			vEnv.User, vEnv.Addr, vEnv.Database, err.Error())
+			connOnline.User, connOnline.Addr, connOnline.Database, err.Error())
 		common.Config.TestDSN.Disable = true
 	}
 
@@ -109,42 +109,50 @@ func BuildEnv() (*VirtualEnv, *database.Connector) {
 }
 
 // RealDB 从测试环境中获取通过hash后的DB
-func (ve VirtualEnv) RealDB(hash string) string {
-	if _, ok := ve.hash2Db[hash]; ok {
-		return ve.hash2Db[hash]
+func (vEnv *VirtualEnv) RealDB(hash string) string {
+	if _, ok := vEnv.Hash2DB[hash]; ok {
+		return vEnv.Hash2DB[hash]
+	}
+	// hash may be real database name not hash
+	if strings.HasPrefix(hash, "optimizer_") {
+		common.Log.Warning("RealDB, Hash2DB missing hash map: %s", hash)
 	}
 	return hash
 }
 
 // DBHash 从测试环境中根据DB找到对应的hash值
-func (ve VirtualEnv) DBHash(db string) string {
-	if _, ok := ve.DBRef[db]; ok {
-		return ve.DBRef[db]
+func (vEnv *VirtualEnv) DBHash(db string) string {
+	if _, ok := vEnv.DBRef[db]; ok {
+		return vEnv.DBRef[db]
 	}
 	return db
 }
 
 // CleanUp 环境清理
-func (ve VirtualEnv) CleanUp() bool {
+func (vEnv *VirtualEnv) CleanUp() bool {
 	if !common.Config.TestDSN.Disable && common.Config.DropTestTemporary {
 		common.Log.Debug("CleanUp ...")
-		for db := range ve.hash2Db {
-			ve.Database = db
-			_, err := ve.Query(fmt.Sprintf("drop database %s", db))
+		for db := range vEnv.Hash2DB {
+			_, err := vEnv.Query(fmt.Sprintf("drop database %s", db))
 			if err != nil {
 				common.Log.Error("CleanUp failed Error: %s", err)
 				return false
 			}
 		}
+		// cleanup hash map
+		vEnv.DBRef = make(map[string]string)
+		vEnv.Hash2DB = make(map[string]string)
+		vEnv.TableMap = make(map[string]map[string]string)
+
 		common.Log.Debug("CleanUp, done")
 	}
 	return true
 }
 
 // CleanupTestDatabase 清除一小时前的环境
-func (ve *VirtualEnv) CleanupTestDatabase() {
+func (vEnv *VirtualEnv) CleanupTestDatabase() {
 	common.Log.Debug("CleanupTestDatabase ...")
-	dbs, err := ve.Query("show databases like 'optimizer%%'")
+	dbs, err := vEnv.Query("show databases like 'optimizer%%'")
 	if err != nil {
 		common.Log.Error("CleanupTestDatabase failed Error:%s", err.Error())
 		return
@@ -172,7 +180,7 @@ func (ve *VirtualEnv) CleanupTestDatabase() {
 
 		subHour := time.Since(pastTime).Hours()
 		if subHour > float64(minHour) {
-			if _, err := ve.Query(fmt.Sprintf("drop database %s", testDatabase)); err != nil {
+			if _, err := vEnv.Query(fmt.Sprintf("drop database %s", testDatabase)); err != nil {
 				common.Log.Error("CleanupTestDatabase failed Error: %s", err.Error())
 				continue
 			}
@@ -188,14 +196,14 @@ func (ve *VirtualEnv) CleanupTestDatabase() {
 
 // BuildVirtualEnv rEnv为SQL源环境，DB使用的信息从接口获取
 // 注意：如果是USE,DDL等语句，执行完第一条就会返回，后面的SQL不会执行
-func (ve *VirtualEnv) BuildVirtualEnv(rEnv *database.Connector, SQLs ...string) bool {
+func (vEnv *VirtualEnv) BuildVirtualEnv(rEnv *database.Connector, SQLs ...string) bool {
 	var stmt sqlparser.Statement
 	var err error
 
 	// 置空错误信息
-	ve.Error = nil
+	vEnv.Error = nil
 	// 检测是否已经创建初始数据库，如果未创建则创建一个名称hash过的映射数据库
-	err = ve.createDatabase(rEnv, rEnv.Database)
+	err = vEnv.createDatabase(rEnv)
 	common.LogIfWarn(err, "")
 
 	// 测试环境检测
@@ -232,15 +240,14 @@ func (ve *VirtualEnv) BuildVirtualEnv(rEnv *database.Connector, SQLs ...string)
 				rEnv.Database = stmt.DBName.String()
 
 				// use DB 后检查 DB是否已经创建，如果没有创建则创建DB
-				err = ve.createDatabase(rEnv, rEnv.Database)
+				err = vEnv.createDatabase(rEnv)
 				common.LogIfWarn(err, "")
 			}
 			return true
 		case *sqlparser.DDL:
 			// 如果是DDL，则先获取DDL对应的表结构，然后直接在测试环境接执行SQL
 			// 为不影响其他SQL操作，复制一个Connector对象，将数据库切换到对应的DB上直接执行
-			tmpDB := *ve.Connector
-			tmpDB.Database = ve.DBRef[rEnv.Database]
+			vEnv.Database = vEnv.DBRef[rEnv.Database]
 
 			// 为了支持并发，需要将DB进行映射，但db.table这种形式无法保证DB的映射是正确的
 			// TODO：暂不支持 create db.tableName (id int) 形式的建表语句
@@ -266,7 +273,7 @@ func (ve *VirtualEnv) BuildVirtualEnv(rEnv *database.Connector, SQLs ...string)
 			// 拉取表结构
 			table := stmt.Table.Name.String()
 			if table != "" {
-				err = ve.createTable(rEnv, rEnv.Database, table)
+				err = vEnv.createTable(rEnv, table)
 				// 这里如果报错可能有两种可能：
 				// 1. SQL 是 Create 语句，线上环境并没有相关的库表结构
 				// 2. 在测试环境中执行 SQL 报错
@@ -274,16 +281,16 @@ func (ve *VirtualEnv) BuildVirtualEnv(rEnv *database.Connector, SQLs ...string)
 				// 如果是因为执行 SQL 报错，那么其他地方执行 SQL 的时候也一定会报错
 				// 所以这里不需要 `return false`，可以继续执行
 				if err != nil {
-					common.Log.Error("BuildVirtualEnv Error : %v", err)
+					common.Log.Warning("BuildVirtualEnv Error : %v", err)
 				}
 			}
 
-			_, err = tmpDB.Query(sql)
+			_, err = vEnv.Query(sql)
 			if err != nil {
 				switch stmt.Action {
 				case "create", "alter":
 					// 如果是创建或者修改语句，且报错信息为如重复建表、重复索引等信息，将错误反馈到上一次层输出建议
-					ve.Error = err
+					vEnv.Error = err
 				default:
 					common.Log.Error("BuildVirtualEnv DDL Execute Error : %v", err)
 				}
@@ -298,8 +305,7 @@ func (ve *VirtualEnv) BuildVirtualEnv(rEnv *database.Connector, SQLs ...string)
 			if db == "" {
 				db = rEnv.Database
 			}
-			tmpEnv := rEnv
-			tmpEnv.Database = db
+			rEnv.Database = db
 
 			// 创建数据库环境
 			for _, tb := range table.Table {
@@ -308,8 +314,8 @@ func (ve *VirtualEnv) BuildVirtualEnv(rEnv *database.Connector, SQLs ...string)
 				}
 
 				// 视图检查
-				common.Log.Debug("BuildVirtualEnv Checking view -- %s.%s", tmpEnv.Database, tb.TableName)
-				tbStatus, err := tmpEnv.ShowTableStatus(tb.TableName)
+				common.Log.Debug("BuildVirtualEnv Checking view -- %s.%s", rEnv.Database, tb.TableName)
+				tbStatus, err := rEnv.ShowTableStatus(tb.TableName)
 				if err != nil {
 					common.Log.Error("BuildVirtualEnv ShowTableStatus Error : %v", err)
 					return false
@@ -317,9 +323,8 @@ func (ve *VirtualEnv) BuildVirtualEnv(rEnv *database.Connector, SQLs ...string)
 
 				// 如果是视图，解析语句
 				if len(tbStatus.Rows) > 0 && string(tbStatus.Rows[0].Comment) == "VIEW" {
-					tmpEnv.Database = db
 					var viewDDL string
-					viewDDL, err = tmpEnv.ShowCreateTable(tb.TableName)
+					viewDDL, err = rEnv.ShowCreateTable(tb.TableName)
 					if err != nil {
 						common.Log.Error("BuildVirtualEnv create view failed: %v", err)
 						return false
@@ -331,14 +336,14 @@ func (ve *VirtualEnv) BuildVirtualEnv(rEnv *database.Connector, SQLs ...string)
 						return false
 					}
 					viewDDL = viewDDL[startIdx+2:]
-					if !ve.BuildVirtualEnv(tmpEnv, viewDDL) {
+					if !vEnv.BuildVirtualEnv(rEnv, viewDDL) {
 						return false
 					}
 				}
 
-				err = ve.createTable(tmpEnv, db, tb.TableName)
+				err = vEnv.createTable(rEnv, tb.TableName)
 				if err != nil {
-					common.Log.Error("BuildVirtualEnv %s.%s Error : %v", db, tb.TableName, err)
+					common.Log.Error("BuildVirtualEnv %s.%s Error : %v", rEnv.Database, tb.TableName, err)
 					return false
 				}
 			}
@@ -347,36 +352,39 @@ func (ve *VirtualEnv) BuildVirtualEnv(rEnv *database.Connector, SQLs ...string)
 	return true
 }
 
-func (ve VirtualEnv) createDatabase(rEnv *database.Connector, dbName string) error {
+func (vEnv *VirtualEnv) createDatabase(rEnv *database.Connector) error {
 	// 生成映射关系
-	if _, ok := ve.DBRef[dbName]; ok {
-		common.Log.Debug("createDatabase, Database `%s` created", dbName)
+	if _, ok := vEnv.DBRef[rEnv.Database]; ok {
+		common.Log.Debug("createDatabase, Database `%s` has created, mapping from `%s`", vEnv.DBRef[rEnv.Database], rEnv.Database)
 		return nil
 	}
 
 	// optimizer_YYMMDDHHmmss_xxxx
-	dbHash := fmt.Sprintf("optimizer_%s_%s", time.Now().Format("060102150405"), uniuri.New())
-	common.Log.Debug("createDatabase, mapping `%s` :`%s`-->`%s`", dbName, dbName, dbHash)
-	ddl, err := rEnv.ShowCreateDatabase(dbName)
+	dbHash := fmt.Sprintf("optimizer_%s_%s", // Total 39 bytes
+		time.Now().Format("060102150405"), // 12 Bytes 20180102030405
+		strings.ToLower(uniuri.New()))     // 16 Bytes random string
+	common.Log.Debug("createDatabase, mapping `%s` :`%s`-->`%s`", rEnv.Database, rEnv.Database, dbHash)
+	ddl, err := rEnv.ShowCreateDatabase(rEnv.Database)
 	if err != nil {
 		common.Log.Warning("createDatabase, rEnv.ShowCreateDatabase Error : %v", err)
-		ddl = fmt.Sprintf("create database `%s` character set utf8mb4", dbName)
+		ddl = fmt.Sprintf("create database `%s` character set utf8mb4", rEnv.Database)
 	}
 
-	ddl = strings.Replace(ddl, dbName, dbHash, -1)
+	ddl = strings.Replace(ddl, rEnv.Database, dbHash, -1)
 	if ddl == "" {
-		return fmt.Errorf("dbName: '%s' get create info error", dbName)
+		return fmt.Errorf("dbName: '%s' get create info error", rEnv.Database)
 	}
-	res, err := ve.Query(ddl)
+	res, err := vEnv.Query(ddl)
 	if err != nil {
 		common.Log.Warning("createDatabase, Error : %v", err)
 		return err
 	}
-	res.Rows.Close()
+	err = res.Rows.Close()
+	common.LogIfWarn(err, "")
 
 	// 创建成功，添加映射记录
-	ve.DBRef[dbName] = dbHash
-	ve.hash2Db[dbHash] = dbName
+	vEnv.DBRef[rEnv.Database] = dbHash
+	vEnv.Hash2DB[dbHash] = rEnv.Database
 	return nil
 }
 
@@ -400,16 +408,18 @@ func (ve VirtualEnv) createDatabase(rEnv *database.Connector, dbName string) err
 		soar 能够做出判断并进行 session 级别的修改，但是这一阶段可用性保证应该是由用户提供两个完全相同（或测试环境兼容线上环境）
 		的数据库环境来实现的。
 */
-func (ve VirtualEnv) createTable(rEnv *database.Connector, dbName, tbName string) error {
-
-	if dbName == "" {
-		dbName = rEnv.Database
+func (vEnv *VirtualEnv) createTable(rEnv *database.Connector, tbName string) error {
+	// 判断数据库是否已经创建
+	if vEnv.DBRef[rEnv.Database] == "" {
+		// 若没创建，则创建数据库
+		err := vEnv.createDatabase(rEnv)
+		if err != nil {
+			return err
+		}
 	}
-	// 如果 dbName 不为空，说明指定了DB，临时修改rEnv中DB参数，来确保执行正确性
-	rEnv.Database = dbName
 
-	if ve.TableMap[dbName] == nil {
-		ve.TableMap[dbName] = make(map[string]string)
+	if vEnv.TableMap[rEnv.Database] == nil {
+		vEnv.TableMap[rEnv.Database] = make(map[string]string)
 	}
 
 	if strings.ToLower(tbName) == "dual" {
@@ -417,30 +427,20 @@ func (ve VirtualEnv) createTable(rEnv *database.Connector, dbName, tbName string
 		return nil
 	}
 
-	if ve.TableMap[dbName][tbName] != "" {
-		common.Log.Debug("createTable, `%s`.`%s` created", dbName, tbName)
+	if vEnv.TableMap[rEnv.Database][tbName] != "" {
+		common.Log.Debug("createTable, `%s`.`%s` has created, mapping from `%s`.`%s`", vEnv.DBRef[rEnv.Database], tbName, rEnv.Database, tbName)
 		return nil
 	}
 
-	common.Log.Debug("createTable, Database: %s, TableName: %s", dbName, tbName)
+	common.Log.Debug("createTable, Database: %s, TableName: %s", vEnv.DBRef[rEnv.Database], tbName)
 
 	//  TODO：查看是否有外键关联（done），对外键的支持 (未解决循环依赖的问题)
 
-	// 判断数据库是否已经创建
-	if ve.DBRef[dbName] == "" {
-		// 若没创建，则创建数据库
-		err := ve.createDatabase(rEnv, dbName)
-		if err != nil {
-			return err
-		}
-	}
-
 	// 记录Table创建信息
-	ve.TableMap[dbName][tbName] = tbName
+	vEnv.TableMap[rEnv.Database][tbName] = tbName
 
 	// 生成建表语句
 	common.Log.Debug("createTable DSN(%s/%s): generate ddl", rEnv.Addr, rEnv.Database)
-
 	ddl, err := rEnv.ShowCreateTable(tbName)
 	if err != nil {
 		// 有可能是用户新建表，因此线上环境查不到
@@ -449,25 +449,26 @@ func (ve VirtualEnv) createTable(rEnv *database.Connector, dbName, tbName string
 	}
 
 	// 改变数据环境
-	ve.Database = ve.DBRef[dbName]
-	res, err := ve.Query(ddl)
+	vEnv.Database = vEnv.DBRef[rEnv.Database]
+	res, err := vEnv.Query(ddl)
 	if err != nil {
 		// 有可能是用户新建表，因此线上环境查不到
 		common.Log.Error("createTable: %s Error : %v", tbName, err)
 		return err
 	}
-	res.Rows.Close()
+	err = res.Rows.Close()
+	common.LogIfWarn(err, "")
 
 	// 泵取数据
 	if common.Config.Sampling {
-		common.Log.Debug("createTable, Start Sampling data from %s.%s to %s.%s ...", dbName, tbName, ve.DBRef[dbName], tbName)
-		err = ve.SamplingData(rEnv, dbName, tbName)
+		common.Log.Debug("createTable, Start Sampling data from %s.%s to %s.%s ...", rEnv.Database, tbName, vEnv.DBRef[rEnv.Database], tbName)
+		err = vEnv.SamplingData(rEnv, tbName)
 	}
 	return err
 }
 
 // GenTableColumns 为 Rewrite 提供的结构体初始化
-func (ve *VirtualEnv) GenTableColumns(meta common.Meta) common.TableColumns {
+func (vEnv *VirtualEnv) GenTableColumns(meta common.Meta) common.TableColumns {
 	tableColumns := make(common.TableColumns)
 	for dbName, db := range meta {
 		for _, tb := range db.Table {
@@ -475,7 +476,7 @@ func (ve *VirtualEnv) GenTableColumns(meta common.Meta) common.TableColumns {
 			if tb == nil {
 				break
 			}
-			td, err := ve.Connector.ShowColumns(tb.TableName)
+			td, err := vEnv.Connector.ShowColumns(tb.TableName)
 			if err != nil {
 				common.Log.Warn("GenTableColumns, ShowColumns Error: " + err.Error())
 				break
@@ -483,7 +484,7 @@ func (ve *VirtualEnv) GenTableColumns(meta common.Meta) common.TableColumns {
 
 			// tableColumns 初始化
 			if dbName == "" {
-				dbName = ve.RealDB(ve.Connector.Database)
+				dbName = vEnv.RealDB(vEnv.Connector.Database)
 			}
 
 			if _, ok := tableColumns[dbName]; !ok {

env/env_test.go

@@ -18,6 +18,7 @@ package env
 
 import (
 	"flag"
+	"fmt"
 	"os"
 	"testing"
 
@@ -28,24 +29,33 @@ import (
 	"github.com/kr/pretty"
 )
 
-var connTest *database.Connector
 var update = flag.Bool("update", false, "update .golden files")
+var vEnv *VirtualEnv
+var rEnv *database.Connector
 
-func init() {
+func TestMain(m *testing.M) {
+	// 初始化 init
 	common.BaseDir = common.DevPath
 	err := common.ParseConfig("")
 	common.LogIfError(err, "init ParseConfig")
 	common.Log.Debug("env_test init")
-	connTest, err = database.NewConnector(common.Config.TestDSN)
-	if err != nil {
-		common.Log.Critical("Test env Error: %v", err)
+	vEnv, rEnv = BuildEnv()
+	if _, err = vEnv.Version(); err != nil {
+		fmt.Println(err.Error(), ", By pass all advisor test cases")
 		os.Exit(0)
 	}
 
-	if _, err := connTest.Version(); err != nil {
-		common.Log.Critical("Test env Error: %v", err)
+	if _, err := rEnv.Version(); err != nil {
+		fmt.Println(err.Error(), ", By pass all advisor test cases")
 		os.Exit(0)
 	}
+
+	// 分割线
+	flag.Parse()
+	m.Run()
+
+	// 环境清理
+	vEnv.CleanUp()
 }
 
 func TestNewVirtualEnv(t *testing.T) {
@@ -53,7 +63,6 @@ func TestNewVirtualEnv(t *testing.T) {
 	testSQL := []string{
 		"create table t(id int,c1 varchar(20),PRIMARY KEY (id));",
 		"alter table t add index `idx_c1`(c1);",
-		"alter table t add index `idx_c1`(c1);",
 		"select * from city where country_id = 44;",
 		"select * from address where address2 is not null;",
 		"select * from address where address2 is null;",
@@ -95,14 +104,10 @@ func TestNewVirtualEnv(t *testing.T) {
 		"select ID,name from (select address from customer_list where SID=1 order by phone limit 50,10) a join customer_list l on (a.address=l.address) join city c on (c.city=l.city) order by phone desc;",
 	}
 
-	rEnv := connTest
-
-	env := NewVirtualEnv(connTest)
-	defer env.CleanUp()
-	common.GoldenDiff(func() {
+	err := common.GoldenDiff(func() {
 		for _, sql := range testSQL {
-			env.BuildVirtualEnv(rEnv, sql)
-			switch err := env.Error.(type) {
+			vEnv.BuildVirtualEnv(rEnv, sql)
+			switch err := vEnv.Error.(type) {
 			case nil:
 				pretty.Println(sql, "OK")
 			case error:
@@ -118,12 +123,14 @@ func TestNewVirtualEnv(t *testing.T) {
 			}
 		}
 	}, t.Name(), update)
+	if err != nil {
+		t.Error(err)
+	}
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
 
 func TestCleanupTestDatabase(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
-	vEnv, _ := BuildEnv()
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	if common.Config.TestDSN.Disable {
 		common.Log.Warn("common.Config.TestDSN.Disable=true, by pass TestCleanupTestDatabase")
 		return
@@ -153,9 +160,7 @@ func TestCleanupTestDatabase(t *testing.T) {
 }
 
 func TestGenTableColumns(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
-	vEnv, rEnv := BuildEnv()
-	defer vEnv.CleanUp()
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 
 	pretty.Println(common.Config.TestDSN.Disable)
 	if common.Config.TestDSN.Disable {
@@ -223,12 +228,12 @@ func TestGenTableColumns(t *testing.T) {
 }
 
 func TestCreateTable(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
 	orgSamplingCondition := common.Config.SamplingCondition
 	common.Config.SamplingCondition = "LIMIT 1"
 
-	vEnv, rEnv := BuildEnv()
-	defer vEnv.CleanUp()
+	orgREnvDatabase := rEnv.Database
+	rEnv.Database = "sakila"
 	// TODO: support VIEW,
 	tables := []string{
 		"actor",
@@ -256,20 +261,21 @@ func TestCreateTable(t *testing.T) {
 		"store",
 	}
 	for _, table := range tables {
-		err := vEnv.createTable(rEnv, "sakila", table)
+		err := vEnv.createTable(rEnv, table)
 		if err != nil {
 			t.Error(err)
 		}
 	}
 	common.Config.SamplingCondition = orgSamplingCondition
+	rEnv.Database = orgREnvDatabase
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
 
 func TestCreateDatabase(t *testing.T) {
-	common.Log.Debug("Enter function: %s", common.GetFunctionName())
-	vEnv, rEnv := BuildEnv()
-	defer vEnv.CleanUp()
-	err := vEnv.createDatabase(rEnv, "sakila")
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	orgREnvDatabase := rEnv.Database
+	rEnv.Database = "sakila"
+	err := vEnv.createDatabase(rEnv)
 	if err != nil {
 		t.Error(err)
 	}
@@ -280,5 +286,6 @@ func TestCreateDatabase(t *testing.T) {
 	if vEnv.DBHash("not_exist_db") != "not_exist_db" {
 		t.Errorf("database: not_exist_db rehashed!")
 	}
+	rEnv.Database = orgREnvDatabase
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }

env/testdata/TestNewVirtualEnv.golden

 create table t(id int,c1 varchar(20),PRIMARY KEY (id)); OK
 alter table t add index `idx_c1`(c1); OK
-alter table t add index `idx_c1`(c1); OK
 select * from city where country_id = 44; OK
 select * from address where address2 is not null; OK
 select * from address where address2 is null; OK

vendor/github.com/pingcap/parser/ast/ast.go

@@ -25,6 +25,8 @@ import (
 // Node is the basic element of the AST.
 // Interfaces embed Node should have 'Node' name suffix.
 type Node interface {
+	// Restore returns the sql text from ast tree
+	Restore(ctx *RestoreCtx) error
 	// Accept accepts Visitor to visit itself.
 	// The returned node should replace original node.
 	// ok returns false to stop visiting.

vendor/github.com/pingcap/parser/ast/ddl.go

@@ -14,6 +14,8 @@
 package ast
 
 import (
+	"github.com/pingcap/errors"
+	"github.com/pingcap/parser/auth"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/types"
 )
@@ -61,6 +63,23 @@ type DatabaseOption struct {
 	Value string
 }
 
+// Restore implements Node interface.
+func (n *DatabaseOption) Restore(ctx *RestoreCtx) error {
+	switch n.Tp {
+	case DatabaseOptionCharset:
+		ctx.WriteKeyWord("CHARACTER SET")
+		ctx.WritePlain(" = ")
+		ctx.WritePlain(n.Value)
+	case DatabaseOptionCollate:
+		ctx.WriteKeyWord("COLLATE")
+		ctx.WritePlain(" = ")
+		ctx.WritePlain(n.Value)
+	default:
+		return errors.Errorf("invalid DatabaseOptionType: %d", n.Tp)
+	}
+	return nil
+}
+
 // CreateDatabaseStmt is a statement to create a database.
 // See https://dev.mysql.com/doc/refman/5.7/en/create-database.html
 type CreateDatabaseStmt struct {
@@ -71,6 +90,23 @@ type CreateDatabaseStmt struct {
 	Options     []*DatabaseOption
 }
 
+// Restore implements Node interface.
+func (n *CreateDatabaseStmt) Restore(ctx *RestoreCtx) error {
+	ctx.WriteKeyWord("CREATE DATABASE ")
+	if n.IfNotExists {
+		ctx.WriteKeyWord("IF NOT EXISTS ")
+	}
+	ctx.WriteName(n.Name)
+	for _, option := range n.Options {
+		ctx.WritePlain(" ")
+		err := option.Restore(ctx)
+		if err != nil {
+			return errors.Trace(err)
+		}
+	}
+	return nil
+}
+
 // Accept implements Node Accept interface.
 func (n *CreateDatabaseStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -90,6 +126,16 @@ type DropDatabaseStmt struct {
 	Name     string
 }
 
+// Restore implements Node interface.
+func (n *DropDatabaseStmt) Restore(ctx *RestoreCtx) error {
+	ctx.WriteKeyWord("DROP DATABASE ")
+	if n.IfExists {
+		ctx.WriteKeyWord("IF EXISTS ")
+	}
+	ctx.WriteName(n.Name)
+	return nil
+}
+
 // Accept implements Node Accept interface.
 func (n *DropDatabaseStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -108,6 +154,11 @@ type IndexColName struct {
 	Length int
 }
 
+// Restore implements Node interface.
+func (n *IndexColName) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *IndexColName) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -134,6 +185,11 @@ type ReferenceDef struct {
 	OnUpdate      *OnUpdateOpt
 }
 
+// Restore implements Node interface.
+func (n *ReferenceDef) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *ReferenceDef) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -199,6 +255,11 @@ type OnDeleteOpt struct {
 	ReferOpt ReferOptionType
 }
 
+// Restore implements Node interface.
+func (n *OnDeleteOpt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *OnDeleteOpt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -215,6 +276,11 @@ type OnUpdateOpt struct {
 	ReferOpt ReferOptionType
 }
 
+// Restore implements Node interface.
+func (n *OnUpdateOpt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *OnUpdateOpt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -259,6 +325,11 @@ type ColumnOption struct {
 	Refer *ReferenceDef
 }
 
+// Restore implements Node interface.
+func (n *ColumnOption) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *ColumnOption) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -290,6 +361,34 @@ type IndexOption struct {
 	Comment      string
 }
 
+// Restore implements Node interface.
+func (n *IndexOption) Restore(ctx *RestoreCtx) error {
+	hasPrevOption := false
+	if n.KeyBlockSize > 0 {
+		ctx.WriteKeyWord("KEY_BLOCK_SIZE")
+		ctx.WritePlainf("=%d", n.KeyBlockSize)
+		hasPrevOption = true
+	}
+
+	if n.Tp != model.IndexTypeInvalid {
+		if hasPrevOption {
+			ctx.WritePlain(" ")
+		}
+		ctx.WriteKeyWord("USING ")
+		ctx.WritePlain(n.Tp.String())
+		hasPrevOption = true
+	}
+
+	if n.Comment != "" {
+		if hasPrevOption {
+			ctx.WritePlain(" ")
+		}
+		ctx.WriteKeyWord("COMMENT ")
+		ctx.WriteString(n.Comment)
+	}
+	return nil
+}
+
 // Accept implements Node Accept interface.
 func (n *IndexOption) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -330,6 +429,11 @@ type Constraint struct {
 	Option *IndexOption // Index Options
 }
 
+// Restore implements Node interface.
+func (n *Constraint) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *Constraint) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -370,6 +474,11 @@ type ColumnDef struct {
 	Options []*ColumnOption
 }
 
+// Restore implements Node interface.
+func (n *ColumnDef) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *ColumnDef) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -408,6 +517,11 @@ type CreateTableStmt struct {
 	Select      ResultSetNode
 }
 
+// Restore implements Node interface.
+func (n *CreateTableStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *CreateTableStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -459,6 +573,12 @@ type DropTableStmt struct {
 
 	IfExists bool
 	Tables   []*TableName
+	IsView   bool
+}
+
+// Restore implements Node interface.
+func (n *DropTableStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
 }
 
 // Accept implements Node Accept interface.
@@ -491,6 +611,11 @@ type RenameTableStmt struct {
 	TableToTables []*TableToTable
 }
 
+// Restore implements Node interface.
+func (n *RenameTableStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *RenameTableStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -527,6 +652,11 @@ type TableToTable struct {
 	NewTable *TableName
 }
 
+// Restore implements Node interface.
+func (n *TableToTable) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *TableToTable) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -552,16 +682,39 @@ func (n *TableToTable) Accept(v Visitor) (Node, bool) {
 type CreateViewStmt struct {
 	ddlNode
 
-	OrReplace bool
-	ViewName  *TableName
-	Cols      []model.CIStr
-	Select    StmtNode
+	OrReplace   bool
+	ViewName    *TableName
+	Cols        []model.CIStr
+	Select      StmtNode
+	Algorithm   model.ViewAlgorithm
+	Definer     *auth.UserIdentity
+	Security    model.ViewSecurity
+	CheckOption model.ViewCheckOption
+}
+
+// Restore implements Node interface.
+func (n *CreateViewStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
 }
 
 // Accept implements Node Accept interface.
 func (n *CreateViewStmt) Accept(v Visitor) (Node, bool) {
-	// TODO: implement the details.
-	return n, true
+	newNode, skipChildren := v.Enter(n)
+	if skipChildren {
+		return v.Leave(newNode)
+	}
+	n = newNode.(*CreateViewStmt)
+	node, ok := n.ViewName.Accept(v)
+	if !ok {
+		return n, false
+	}
+	n.ViewName = node.(*TableName)
+	selnode, ok := n.Select.Accept(v)
+	if !ok {
+		return n, false
+	}
+	n.Select = selnode.(*SelectStmt)
+	return v.Leave(n)
 }
 
 // CreateIndexStmt is a statement to create an index.
@@ -576,6 +729,11 @@ type CreateIndexStmt struct {
 	IndexOption   *IndexOption
 }
 
+// Restore implements Node interface.
+func (n *CreateIndexStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *CreateIndexStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -615,6 +773,22 @@ type DropIndexStmt struct {
 	Table     *TableName
 }
 
+// Restore implements Node interface.
+func (n *DropIndexStmt) Restore(ctx *RestoreCtx) error {
+	ctx.WriteKeyWord("DROP INDEX ")
+	if n.IfExists {
+		ctx.WriteKeyWord("IF EXISTS ")
+	}
+	ctx.WriteName(n.IndexName)
+	ctx.WriteKeyWord(" ON ")
+
+	if err := n.Table.Restore(ctx); err != nil {
+		return errors.Annotate(err, "An error occurred while add index")
+	}
+
+	return nil
+}
+
 // Accept implements Node Accept interface.
 func (n *DropIndexStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -703,6 +877,11 @@ type ColumnPosition struct {
 	RelativeColumn *ColumnName
 }
 
+// Restore implements Node interface.
+func (n *ColumnPosition) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *ColumnPosition) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -743,8 +922,9 @@ const (
 	AlterTableAddPartitions
 	AlterTableCoalescePartitions
 	AlterTableDropPartition
+	AlterTableTruncatePartition
 
-// TODO: Add more actions
+	// TODO: Add more actions
 )
 
 // LockType is the type for AlterTableSpec.
@@ -779,6 +959,11 @@ type AlterTableSpec struct {
 	Num             uint64
 }
 
+// Restore implements Node interface.
+func (n *AlterTableSpec) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *AlterTableSpec) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -833,6 +1018,11 @@ type AlterTableStmt struct {
 	Specs []*AlterTableSpec
 }
 
+// Restore implements Node interface.
+func (n *AlterTableStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *AlterTableStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -863,6 +1053,11 @@ type TruncateTableStmt struct {
 	Table *TableName
 }
 
+// Restore implements Node interface.
+func (n *TruncateTableStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *TruncateTableStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)

vendor/github.com/pingcap/parser/ast/dml.go

@@ -14,6 +14,7 @@
 package ast
 
 import (
+	"github.com/pingcap/errors"
 	"github.com/pingcap/parser/auth"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/mysql"
@@ -82,6 +83,11 @@ type Join struct {
 	StraightJoin bool
 }
 
+// Restore implements Node interface.
+func (n *Join) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *Join) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -125,6 +131,22 @@ type TableName struct {
 	IndexHints []*IndexHint
 }
 
+// Restore implements Node interface.
+func (n *TableName) Restore(ctx *RestoreCtx) error {
+	if n.Schema.String() != "" {
+		ctx.WriteName(n.Schema.String())
+		ctx.WritePlain(".")
+	}
+	ctx.WriteName(n.Name.String())
+	for _, value := range n.IndexHints {
+		ctx.WritePlain(" ")
+		if err := value.Restore(ctx); err != nil {
+			return errors.Annotate(err, "An error occurred while splicing IndexHints")
+		}
+	}
+	return nil
+}
+
 // IndexHintType is the type for index hint use, ignore or force.
 type IndexHintType int
 
@@ -153,6 +175,47 @@ type IndexHint struct {
 	HintScope  IndexHintScope
 }
 
+// IndexHint Restore (The const field uses switch to facilitate understanding)
+func (n *IndexHint) Restore(ctx *RestoreCtx) error {
+	indexHintType := ""
+	switch n.HintType {
+	case 1:
+		indexHintType = "USE INDEX"
+	case 2:
+		indexHintType = "IGNORE INDEX"
+	case 3:
+		indexHintType = "FORCE INDEX"
+	default: // Prevent accidents
+		return errors.New("IndexHintType has an error while matching")
+	}
+
+	indexHintScope := ""
+	switch n.HintScope {
+	case 1:
+		indexHintScope = ""
+	case 2:
+		indexHintScope = " FOR JOIN"
+	case 3:
+		indexHintScope = " FOR ORDER BY"
+	case 4:
+		indexHintScope = " FOR GROUP BY"
+	default: // Prevent accidents
+		return errors.New("IndexHintScope has an error while matching")
+	}
+	ctx.WriteKeyWord(indexHintType)
+	ctx.WriteKeyWord(indexHintScope)
+	ctx.WritePlain(" (")
+	for i, value := range n.IndexNames {
+		if i > 0 {
+			ctx.WritePlain(", ")
+		}
+		ctx.WriteName(value.O)
+	}
+	ctx.WritePlain(")")
+
+	return nil
+}
+
 // Accept implements Node Accept interface.
 func (n *TableName) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -169,6 +232,11 @@ type DeleteTableList struct {
 	Tables []*TableName
 }
 
+// Restore implements Node interface.
+func (n *DeleteTableList) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *DeleteTableList) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -195,6 +263,11 @@ type OnCondition struct {
 	Expr ExprNode
 }
 
+// Restore implements Node interface.
+func (n *OnCondition) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *OnCondition) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -222,6 +295,11 @@ type TableSource struct {
 	AsName model.CIStr
 }
 
+// Restore implements Node interface.
+func (n *TableSource) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *TableSource) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -268,6 +346,11 @@ type WildCardField struct {
 	Schema model.CIStr
 }
 
+// Restore implements Node interface.
+func (n *WildCardField) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *WildCardField) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -298,6 +381,11 @@ type SelectField struct {
 	Auxiliary bool
 }
 
+// Restore implements Node interface.
+func (n *SelectField) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *SelectField) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -322,6 +410,11 @@ type FieldList struct {
 	Fields []*SelectField
 }
 
+// Restore implements Node interface.
+func (n *FieldList) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *FieldList) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -346,6 +439,11 @@ type TableRefsClause struct {
 	TableRefs *Join
 }
 
+// Restore implements Node interface.
+func (n *TableRefsClause) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *TableRefsClause) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -369,6 +467,11 @@ type ByItem struct {
 	Desc bool
 }
 
+// Restore implements Node interface.
+func (n *ByItem) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *ByItem) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -390,6 +493,11 @@ type GroupByClause struct {
 	Items []*ByItem
 }
 
+// Restore implements Node interface.
+func (n *GroupByClause) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *GroupByClause) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -413,6 +521,11 @@ type HavingClause struct {
 	Expr ExprNode
 }
 
+// Restore implements Node interface.
+func (n *HavingClause) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *HavingClause) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -435,6 +548,11 @@ type OrderByClause struct {
 	ForUnion bool
 }
 
+// Restore implements Node interface.
+func (n *OrderByClause) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *OrderByClause) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -488,6 +606,11 @@ type SelectStmt struct {
 	IsInBraces bool
 }
 
+// Restore implements Node interface.
+func (n *SelectStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *SelectStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -582,6 +705,11 @@ type UnionSelectList struct {
 	Selects []*SelectStmt
 }
 
+// Restore implements Node interface.
+func (n *UnionSelectList) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *UnionSelectList) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -610,6 +738,11 @@ type UnionStmt struct {
 	Limit      *Limit
 }
 
+// Restore implements Node interface.
+func (n *UnionStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *UnionStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -650,6 +783,11 @@ type Assignment struct {
 	Expr ExprNode
 }
 
+// Restore implements Node interface.
+func (n *Assignment) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *Assignment) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -684,6 +822,11 @@ type LoadDataStmt struct {
 	IgnoreLines uint64
 }
 
+// Restore implements Node interface.
+func (n *LoadDataStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *LoadDataStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -737,6 +880,11 @@ type InsertStmt struct {
 	Select      ResultSetNode
 }
 
+// Restore implements Node interface.
+func (n *InsertStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *InsertStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -813,6 +961,11 @@ type DeleteStmt struct {
 	TableHints []*TableOptimizerHint
 }
 
+// Restore implements Node interface.
+func (n *DeleteStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *DeleteStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -873,6 +1026,11 @@ type UpdateStmt struct {
 	TableHints    []*TableOptimizerHint
 }
 
+// Restore implements Node interface.
+func (n *UpdateStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *UpdateStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -924,6 +1082,21 @@ type Limit struct {
 	Offset ExprNode
 }
 
+// Restore implements Node interface.
+func (n *Limit) Restore(ctx *RestoreCtx) error {
+	ctx.WriteKeyWord("LIMIT ")
+	if n.Offset != nil {
+		if err := n.Offset.Restore(ctx); err != nil {
+			return errors.Annotate(err, "An error occurred while restore Limit.Offset")
+		}
+		ctx.WritePlain(",")
+	}
+	if err := n.Count.Restore(ctx); err != nil {
+		return errors.Annotate(err, "An error occurred while restore Limit.Count")
+	}
+	return nil
+}
+
 // Accept implements Node Accept interface.
 func (n *Limit) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -1004,6 +1177,11 @@ type ShowStmt struct {
 	Where       ExprNode
 }
 
+// Restore implements Node interface.
+func (n *ShowStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *ShowStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -1064,6 +1242,11 @@ type WindowSpec struct {
 	Frame       *FrameClause
 }
 
+// Restore implements Node interface.
+func (n *WindowSpec) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *WindowSpec) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -1102,6 +1285,11 @@ type PartitionByClause struct {
 	Items []*ByItem
 }
 
+// Restore implements Node interface.
+func (n *PartitionByClause) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *PartitionByClause) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -1138,6 +1326,11 @@ type FrameClause struct {
 	Extent FrameExtent
 }
 
+// Restore implements Node interface.
+func (n *FrameClause) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *FrameClause) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -1186,6 +1379,11 @@ type FrameBound struct {
 	Unit ExprNode
 }
 
+// Restore implements Node interface.
+func (n *FrameBound) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *FrameBound) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)

vendor/github.com/pingcap/parser/ast/functions.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/pingcap/errors"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/types"
 )
@@ -327,6 +328,11 @@ type FuncCallExpr struct {
 	Args []ExprNode
 }
 
+// Restore implements Node interface.
+func (n *FuncCallExpr) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Format the ExprNode into a Writer.
 func (n *FuncCallExpr) Format(w io.Writer) {
 	fmt.Fprintf(w, "%s(", n.FnName.L)
@@ -399,6 +405,11 @@ type FuncCastExpr struct {
 	FunctionType CastFunctionType
 }
 
+// Restore implements Node interface.
+func (n *FuncCastExpr) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Format the ExprNode into a Writer.
 func (n *FuncCastExpr) Format(w io.Writer) {
 	switch n.FunctionType {
@@ -484,6 +495,10 @@ const (
 	AggFuncBitXor = "bit_xor"
 	// AggFuncBitAnd is the name of bit_and function.
 	AggFuncBitAnd = "bit_and"
+	// AggFuncVarPop is the name of var_pop function
+	AggFuncVarPop = "var_pop"
+	// AggFuncVarSamp is the name of var_samp function
+	AggFuncVarSamp = "var_samp"
 	// AggFuncStddevPop is the name of stddev_pop function
 	AggFuncStddevPop = "stddev_pop"
 	// AggFuncStddevSamp is the name of stddev_samp function
@@ -503,6 +518,11 @@ type AggregateFuncExpr struct {
 	Distinct bool
 }
 
+// Restore implements Node interface.
+func (n *AggregateFuncExpr) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Format the ExprNode into a Writer.
 func (n *AggregateFuncExpr) Format(w io.Writer) {
 	panic("Not implemented")
@@ -571,6 +591,11 @@ type WindowFuncExpr struct {
 	Spec WindowSpec
 }
 
+// Restore implements Node interface.
+func (n *WindowFuncExpr) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Format formats the window function expression into a Writer.
 func (n *WindowFuncExpr) Format(w io.Writer) {
 	panic("Not implemented")

vendor/github.com/pingcap/parser/ast/stats.go

@@ -13,7 +13,10 @@
 
 package ast
 
-import "github.com/pingcap/parser/model"
+import (
+	"github.com/pingcap/errors"
+	"github.com/pingcap/parser/model"
+)
 
 var (
 	_ StmtNode = &AnalyzeTableStmt{}
@@ -34,6 +37,11 @@ type AnalyzeTableStmt struct {
 	IndexFlag bool
 }
 
+// Restore implements Node interface.
+func (n *AnalyzeTableStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *AnalyzeTableStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -58,6 +66,16 @@ type DropStatsStmt struct {
 	Table *TableName
 }
 
+// Restore implements Node interface.
+func (n *DropStatsStmt) Restore(ctx *RestoreCtx) error {
+	ctx.WriteKeyWord("DROP STATS ")
+	if err := n.Table.Restore(ctx); err != nil {
+		return errors.Annotate(err, "An error occurred while add table")
+	}
+
+	return nil
+}
+
 // Accept implements Node Accept interface.
 func (n *DropStatsStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)
@@ -80,6 +98,11 @@ type LoadStatsStmt struct {
 	Path string
 }
 
+// Restore implements Node interface.
+func (n *LoadStatsStmt) Restore(ctx *RestoreCtx) error {
+	return errors.New("Not implemented")
+}
+
 // Accept implements Node Accept interface.
 func (n *LoadStatsStmt) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)

vendor/github.com/pingcap/parser/charset/charset.go

@@ -94,6 +94,7 @@ func ValidCharsetAndCollation(cs string, co string) bool {
 	if co == "" {
 		return true
 	}
+	co = strings.ToLower(co)
 	_, ok = c.Collations[co]
 	if !ok {
 		return false