Merge pull request #3 from XiaoMi/master

merge by xiaomi

Makefile

@@ -45,8 +45,9 @@ deps:
 	@echo "\033[92mDependency check\033[0m"
 	@bash ./deps.sh
 	# The retool tools.json is setup from retool-install.sh
+	# some packages download need more open internet access
 	retool sync
-	retool do gometalinter.v2 intall
+	#retool do gometalinter.v2 --install
 
 # Code format
 .PHONY: fmt
@@ -151,7 +152,7 @@ lint: build
 	@echo "gometalinter check your code is pretty good"
 
 .PHONY: release
-release: deps build
+release: build
 	@echo "\033[92mCross platform building for release ...\033[0m"
 	@mkdir -p release
 	@for GOOS in darwin linux windows; do \
@@ -176,7 +177,7 @@ docker:
 	-v `pwd`/doc/example/sakila.sql.gz:/docker-entrypoint-initdb.d/sakila.sql.gz \
 	$(MYSQL_RELEASE):$(MYSQL_VERSION)
 
-	@echo -n "waiting for sakila database initializing "
+	@echo "waiting for sakila database initializing "
 	@while ! mysql -h 127.0.0.1 -u root sakila -p1tIsB1g3rt -NBe "do 1;" 2>/dev/null; do \
 	printf '.' ; \
 	sleep 1 ; \
@@ -201,7 +202,7 @@ daily: | deps fmt vendor docker cover doc lint release install main_test clean l
 
 # vendor, docker will cost long time, if all those are ready, daily-quick will much more fast.
 .PHONY: daily-quick
-daily-quick: | deps fmt cover doc lint logo
+daily-quick: | deps fmt cover main_test doc lint logo
 	@echo "\033[92mdaily-quick build finished\033[0m"
 
 .PHONY: logo

advisor/heuristic.go

@@ -1629,6 +1629,9 @@ func (q *Query4Audit) RuleImpreciseDataType() Rule {
 			case *tidb.CreateTableStmt:
 				// Create table statement
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					switch col.Tp.Tp {
 					case mysql.TypeFloat, mysql.TypeDouble, mysql.TypeDecimal, mysql.TypeNewDecimal:
 						rule = HeuristicRules["COL.009"]
@@ -1641,6 +1644,9 @@ func (q *Query4Audit) RuleImpreciseDataType() Rule {
 					switch spec.Tp {
 					case tidb.AlterTableAddColumns, tidb.AlterTableChangeColumn, tidb.AlterTableModifyColumn:
 						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
 							switch col.Tp.Tp {
 							case mysql.TypeFloat, mysql.TypeDouble,
 								mysql.TypeDecimal, mysql.TypeNewDecimal:
@@ -1686,6 +1692,9 @@ func (q *Query4Audit) RuleValuesInDefinition() Rule {
 			switch node := tiStmt.(type) {
 			case *tidb.CreateTableStmt:
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					switch col.Tp.Tp {
 					case mysql.TypeSet, mysql.TypeEnum, mysql.TypeBit:
 						rule = HeuristicRules["COL.010"]
@@ -1696,6 +1705,9 @@ func (q *Query4Audit) RuleValuesInDefinition() Rule {
 					switch spec.Tp {
 					case tidb.AlterTableAddColumns, tidb.AlterTableChangeColumn, tidb.AlterTableModifyColumn:
 						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
 							switch col.Tp.Tp {
 							case mysql.TypeSet, mysql.TypeEnum, mysql.TypeBit:
 								rule = HeuristicRules["COL.010"]
@@ -2238,6 +2250,9 @@ func (q *Query4Audit) RuleReadablePasswords() Rule {
 			switch node := tiStmt.(type) {
 			case *tidb.CreateTableStmt:
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					switch col.Tp.Tp {
 					case mysql.TypeString, mysql.TypeVarchar, mysql.TypeVarString,
 						mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob:
@@ -2253,6 +2268,9 @@ func (q *Query4Audit) RuleReadablePasswords() Rule {
 					switch spec.Tp {
 					case tidb.AlterTableModifyColumn, tidb.AlterTableChangeColumn, tidb.AlterTableAddColumns:
 						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
 							switch col.Tp.Tp {
 							case mysql.TypeString, mysql.TypeVarchar, mysql.TypeVarString,
 								mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob:
@@ -2426,6 +2444,9 @@ func (q *Query4Audit) RuleVarcharVSChar() Rule {
 			switch node := tiStmt.(type) {
 			case *tidb.CreateTableStmt:
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					switch col.Tp.Tp {
 					// 在 TiDB 的 AST 中，char 和 binary 的 type 都是 mysql.TypeString
 					// 只是 binary 数据类型的 character 和 collate 是 binary
@@ -2439,6 +2460,9 @@ func (q *Query4Audit) RuleVarcharVSChar() Rule {
 					switch spec.Tp {
 					case tidb.AlterTableAddColumns, tidb.AlterTableChangeColumn, tidb.AlterTableModifyColumn:
 						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
 							switch col.Tp.Tp {
 							case mysql.TypeString:
 								rule = HeuristicRules["COL.008"]
@@ -2553,6 +2577,9 @@ func (q *Query4Audit) RuleBLOBNotNull() Rule {
 			switch node := tiStmt.(type) {
 			case *tidb.CreateTableStmt:
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					switch col.Tp.Tp {
 					case mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob:
 						for _, opt := range col.Options {
@@ -2573,6 +2600,9 @@ func (q *Query4Audit) RuleBLOBNotNull() Rule {
 					switch spec.Tp {
 					case tidb.AlterTableAddColumns, tidb.AlterTableModifyColumn, tidb.AlterTableChangeColumn:
 						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
 							switch col.Tp.Tp {
 							case mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob:
 								for _, opt := range col.Options {
@@ -2808,6 +2838,9 @@ func (q *Query4Audit) RuleTimestampDefault() Rule {
 			switch node := tiStmt.(type) {
 			case *tidb.CreateTableStmt:
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					if col.Tp.Tp == mysql.TypeTimestamp {
 						hasDefault := false
 						for _, option := range col.Options {
@@ -2829,6 +2862,9 @@ func (q *Query4Audit) RuleTimestampDefault() Rule {
 						tidb.AlterTableChangeColumn,
 						tidb.AlterTableAlterColumn:
 						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
 							if col.Tp.Tp == mysql.TypeTimestamp {
 								hasDefault := false
 								for _, option := range col.Options {
@@ -2879,6 +2915,9 @@ func (q *Query4Audit) RuleColumnWithCharset() Rule {
 			switch node := tiStmt.(type) {
 			case *tidb.CreateTableStmt:
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					if col.Tp.Charset != "" || col.Tp.Collate != "" {
 						rule = HeuristicRules["COL.014"]
 						break
@@ -2890,6 +2929,9 @@ func (q *Query4Audit) RuleColumnWithCharset() Rule {
 					case tidb.AlterTableAlterColumn, tidb.AlterTableChangeColumn,
 						tidb.AlterTableModifyColumn, tidb.AlterTableAddColumns:
 						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
 							if col.Tp.Charset != "" || col.Tp.Collate != "" {
 								rule = HeuristicRules["COL.014"]
 								break
@@ -3091,6 +3133,9 @@ func (q *Query4Audit) RuleBlobDefaultValue() Rule {
 			switch node := tiStmt.(type) {
 			case *tidb.CreateTableStmt:
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					switch col.Tp.Tp {
 					case mysql.TypeBlob, mysql.TypeMediumBlob, mysql.TypeTinyBlob, mysql.TypeLongBlob:
 						for _, opt := range col.Options {
@@ -3107,6 +3152,9 @@ func (q *Query4Audit) RuleBlobDefaultValue() Rule {
 					case tidb.AlterTableModifyColumn, tidb.AlterTableAlterColumn,
 						tidb.AlterTableChangeColumn, tidb.AlterTableAddColumns:
 						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
 							switch col.Tp.Tp {
 							case mysql.TypeBlob, mysql.TypeMediumBlob, mysql.TypeTinyBlob, mysql.TypeLongBlob:
 								for _, opt := range col.Options {
@@ -3134,6 +3182,9 @@ func (q *Query4Audit) RuleIntPrecision() Rule {
 			switch node := tiStmt.(type) {
 			case *tidb.CreateTableStmt:
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					switch col.Tp.Tp {
 					case mysql.TypeLong:
 						if (col.Tp.Flen < 10 || col.Tp.Flen > 11) && col.Tp.Flen > 0 {
@@ -3154,6 +3205,9 @@ func (q *Query4Audit) RuleIntPrecision() Rule {
 					case tidb.AlterTableAddColumns, tidb.AlterTableChangeColumn,
 						tidb.AlterTableAlterColumn, tidb.AlterTableModifyColumn:
 						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
 							switch col.Tp.Tp {
 							case mysql.TypeLong:
 								if (col.Tp.Flen < 10 || col.Tp.Flen > 11) && col.Tp.Flen > 0 {
@@ -3185,6 +3239,9 @@ func (q *Query4Audit) RuleVarcharLength() Rule {
 			switch node := tiStmt.(type) {
 			case *tidb.CreateTableStmt:
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					switch col.Tp.Tp {
 					case mysql.TypeVarchar, mysql.TypeVarString:
 						if col.Tp.Flen > common.Config.MaxVarcharLength {
@@ -3199,6 +3256,9 @@ func (q *Query4Audit) RuleVarcharLength() Rule {
 					case tidb.AlterTableAddColumns, tidb.AlterTableChangeColumn,
 						tidb.AlterTableAlterColumn, tidb.AlterTableModifyColumn:
 						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
 							switch col.Tp.Tp {
 							case mysql.TypeVarchar, mysql.TypeVarString:
 								if col.Tp.Flen > common.Config.MaxVarcharLength {
@@ -3247,6 +3307,51 @@ func (q *Query4Audit) RuleColumnNotAllowType() Rule {
 	return rule
 }
 
+// RuleTimePrecision COL.019
+func (q *Query4Audit) RuleTimePrecision() Rule {
+	var rule = q.RuleOK()
+
+	switch q.Stmt.(type) {
+	case *sqlparser.DDL:
+		for _, tiStmt := range q.TiStmt {
+			switch node := tiStmt.(type) {
+			case *tidb.CreateTableStmt:
+				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
+					switch col.Tp.Tp {
+					case mysql.TypeDatetime, mysql.TypeTimestamp, mysql.TypeDuration:
+						if col.Tp.Decimal > 0 {
+							rule = HeuristicRules["COL.019"]
+						}
+					}
+				}
+			case *tidb.AlterTableStmt:
+				for _, spec := range node.Specs {
+					switch spec.Tp {
+					case tidb.AlterTableChangeColumn, tidb.AlterTableAlterColumn,
+						tidb.AlterTableModifyColumn, tidb.AlterTableAddColumns:
+						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
+							switch col.Tp.Tp {
+							case mysql.TypeDatetime, mysql.TypeTimestamp, mysql.TypeDuration:
+								if col.Tp.Decimal > 0 {
+									rule = HeuristicRules["COL.019"]
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return rule
+}
+
 // RuleNoOSCKey KEY.002
 func (q *Query4Audit) RuleNoOSCKey() Rule {
 	var rule = q.RuleOK()
@@ -3292,6 +3397,9 @@ func (q *Query4Audit) RuleMaxTextColsCount() Rule {
 			switch node := tiStmt.(type) {
 			case *tidb.CreateTableStmt:
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					switch col.Tp.Tp {
 					case mysql.TypeBlob, mysql.TypeLongBlob, mysql.TypeMediumBlob, mysql.TypeTinyBlob:
 						textColsCount++
@@ -3452,6 +3560,9 @@ func (q *Query4Audit) RuleAutoIncUnsigned() Rule {
 			switch node := tiStmt.(type) {
 			case *tidb.CreateTableStmt:
 				for _, col := range node.Cols {
+					if col.Tp == nil {
+						continue
+					}
 					for _, opt := range col.Options {
 						if opt.Tp == tidb.ColumnOptionAutoIncrement {
 							if !mysql.HasUnsignedFlag(col.Tp.Flag) {
@@ -3471,6 +3582,9 @@ func (q *Query4Audit) RuleAutoIncUnsigned() Rule {
 					case tidb.AlterTableChangeColumn, tidb.AlterTableAlterColumn,
 						tidb.AlterTableModifyColumn, tidb.AlterTableAddColumns:
 						for _, col := range spec.NewColumns {
+							if col.Tp == nil {
+								continue
+							}
 							for _, opt := range col.Options {
 								if opt.Tp == tidb.ColumnOptionAutoIncrement {
 									if !mysql.HasUnsignedFlag(col.Tp.Flag) {

advisor/heuristic_test.go

@@ -3073,6 +3073,47 @@ func TestRuleColumnNotAllowType(t *testing.T) {
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
 
+// COL.019
+func TestRuleTimePrecision(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	sqls := [][]string{
+		// 正面的例子
+		{
+			"CREATE TABLE t1 (t TIME(3), dt DATETIME(6));",
+			"ALTER TABLE t1 add t TIME(3);",
+		},
+		// 反面的例子
+		{
+			"CREATE TABLE t1 (t TIME, dt DATETIME);",
+			"ALTER TABLE t1 add t TIME;",
+		},
+	}
+	for _, sql := range sqls[0] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleTimePrecision()
+			if rule.Item != "COL.019" {
+				t.Error("Rule not match:", rule.Item, "Expect : COL.019")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+
+	for _, sql := range sqls[1] {
+		q, err := NewQuery4Audit(sql)
+		if err == nil {
+			rule := q.RuleTimePrecision()
+			if rule.Item != "OK" {
+				t.Error("Rule not match:", rule.Item, "Expect : OK")
+			}
+		} else {
+			t.Error("sqlparser.Parse Error:", err)
+		}
+	}
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}
+
 // KEY.002
 func TestRuleNoOSCKey(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())

advisor/index.go

@@ -628,7 +628,6 @@ func (idxAdv *IndexAdvisor) buildJoinIndex(meta common.Meta) []IndexInfo {
 		indexColsList := make(map[string]map[string][]*common.Column)
 		for _, col := range IndexCols {
 			mergeIndex(indexColsList, col)
-
 		}
 
 		if common.Config.TestDSN.Disable || common.Config.OnlineDSN.Disable {
@@ -723,6 +722,11 @@ func (idxAdv *IndexAdvisor) buildIndexWithNoEnv(indexList map[string]map[string]
 
 // mergeIndex 将索引用到的列去重后合并到一起
 func mergeIndex(idxList map[string]map[string][]*common.Column, column *common.Column) {
+	// 散粒度低于阈值将不会添加索引
+	if common.Config.MinCardinality/100 > column.Cardinality {
+		return
+	}
+
 	db := column.DB
 	tb := column.Table
 	if idxList[db] == nil {

advisor/index_test.go

@@ -357,6 +357,8 @@ func TestRuleUpdatePrimaryKey(t *testing.T) {
 
 func TestIndexAdvise(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	minCardinalityBak := common.Config.MinCardinality
+	common.Config.MinCardinality = 20
 	vEnv, rEnv := env.BuildEnv()
 	defer vEnv.CleanUp()
 
@@ -377,12 +379,13 @@ func TestIndexAdvise(t *testing.T) {
 			if idxAdvisor != nil {
 				rule := idxAdvisor.IndexAdvise().Format()
 				if len(rule) > 0 {
-					pretty.Println(rule)
+					_, _ = pretty.Println(rule)
 				}
 			}
 		}
 	}
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+	common.Config.MinCardinality = minCardinalityBak
 }
 
 func TestIndexAdviseNoEnv(t *testing.T) {

advisor/rules.go

@@ -569,6 +569,14 @@ func init() {
 			Case:     "CREATE TABLE tab (a BOOLEAN);",
 			Func:     (*Query4Audit).RuleColumnNotAllowType,
 		},
+		"COL.019": {
+			Item:     "COL.019",
+			Severity: "L1",
+			Summary:  "不建议使用精度在秒级以下的时间数据类型",
+			Content:  "使用高精度的时间数据类型带来的存储空间消耗相对较大；MySQL 在5.6.4以上才可以支持精确到微秒的时间数据类型，使用时需要考虑版本兼容问题。",
+			Case:     "CREATE TABLE t1 (t TIME(3), dt DATETIME(6));",
+			Func:     (*Query4Audit).RuleTimePrecision,
+		},
 		"DIS.001": {
 			Item:     "DIS.001",
 			Severity: "L1",

advisor/rules_test.go

@@ -18,6 +18,7 @@ package advisor
 
 import (
 	"flag"
+	"strings"
 	"testing"
 
 	"github.com/XiaoMi/soar/common"
@@ -26,6 +27,11 @@ import (
 var update = flag.Bool("update", false, "update .golden files")
 
 func TestListTestSQLs(t *testing.T) {
+	for _, sql := range common.TestSQLs {
+		if !strings.HasSuffix(sql, ";") {
+			t.Errorf("%s should end with ';'", sql)
+		}
+	}
 	err := common.GoldenDiff(func() { ListTestSQLs() }, t.Name(), update)
 	if nil != err {
 		t.Fatal(err)

advisor/testdata/TestListHeuristicRules.golden

@@ -532,6 +532,16 @@ CREATE TABLE tab (a varchar(3500));
 ```sql
 CREATE TABLE tab (a BOOLEAN);
 ```
+## 不建议使用精度在秒级以下的时间数据类型
+
+* **Item**:COL.019
+* **Severity**:L1
+* **Content**:使用高精度的时间数据类型带来的存储空间消耗相对较大；MySQL 在5.6.4以上才可以支持精确到微秒的时间数据类型，使用时需要考虑版本兼容问题。
+* **Case**:
+
+```sql
+CREATE TABLE t1 (t TIME(3), dt DATETIME(6));
+```
 ## 消除不必要的 DISTINCT 条件
 
 * **Item**:DIS.001

advisor/testdata/TestListTestSQLs.golden

@@ -79,6 +79,7 @@ SELECT description FROM film WHERE description IN('NEWS','asd') GROUP BY descrip
 alter table address add index idx_city_id(city_id);
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
-SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
 create table hello.t (id int unsigned);
-select * from tb where data >= ''
+select * from tb where data >= '';
+alter table tb alter column id drop default;

advisor/testdata/TestMergeConflictHeuristicRules.golden

@@ -50,6 +50,7 @@ advisor.Rule{Item:"COL.015", Severity:"L4", Summary:"TEXT 和 BLOB 类型的字
 advisor.Rule{Item:"COL.016", Severity:"L1", Summary:"整型定义建议采用 INT(10) 或 BIGINT(20)", Content:"INT(M) 在 integer 数据类型中，M 表示最大显示宽度。 在 INT(M) 中，M 的值跟 INT(M) 所占多少存储空间并无任何关系。 INT(3)、INT(4)、INT(8) 在磁盘上都是占用 4 bytes 的存储空间。", Case:"CREATE TABLE tab (a INT(1));", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.017", Severity:"L2", Summary:"VARCHAR 定义长度过长", Content:"varchar 是可变长字符串，不预先分配存储空间，长度不要超过1024，如果存储长度过长 MySQL 将定义字段类型为 text，独立出来一张表，用主键来对应，避免影响其它字段索引效率。", Case:"CREATE TABLE tab (a varchar(3500));", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"COL.018", Severity:"L1", Summary:"建表语句中使用了不推荐的字段类型", Content:"以下字段类型不被推荐使用：boolean", Case:"CREATE TABLE tab (a BOOLEAN);", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
+advisor.Rule{Item:"COL.019", Severity:"L1", Summary:"不建议使用精度在秒级以下的时间数据类型", Content:"使用高精度的时间数据类型带来的存储空间消耗相对较大；MySQL 在5.6.4以上才可以支持精确到微秒的时间数据类型，使用时需要考虑版本兼容问题。", Case:"CREATE TABLE t1 (t TIME(3), dt DATETIME(6));", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"DIS.001", Severity:"L1", Summary:"消除不必要的 DISTINCT 条件", Content:"太多DISTINCT条件是复杂的裹脚布式查询的症状。考虑将复杂查询分解成许多简单的查询，并减少DISTINCT条件的数量。如果主键列是列的结果集的一部分，则DISTINCT条件可能没有影响。", Case:"SELECT DISTINCT c.c_id,count(DISTINCT c.c_name),count(DISTINCT c.c_e),count(DISTINCT c.c_n),count(DISTINCT c.c_me),c.c_d FROM (select distinct id, name from B) as e WHERE e.country_id = c.country_id", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"DIS.002", Severity:"L3", Summary:"COUNT(DISTINCT) 多列时结果可能和你预想的不同", Content:"COUNT(DISTINCT col) 计算该列除NULL之外的不重复行数，注意 COUNT(DISTINCT col, col2) 如果其中一列全为 NULL 那么即使另一列有不同的值，也返回0。", Case:"SELECT COUNT(DISTINCT col, col2) FROM tbl;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}
 advisor.Rule{Item:"DIS.003", Severity:"L3", Summary:"DISTINCT * 对有主键的表没有意义", Content:"当表已经有主键时，对所有列进行 DISTINCT 的输出结果与不进行 DISTINCT 操作的结果相同，请不要画蛇添足。", Case:"SELECT DISTINCT * FROM film;", Position:0, Func:func(*advisor.Query4Audit) advisor.Rule {...}}

ast/testdata/TestCompress.golden

@@ -160,9 +160,11 @@ alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
-SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
-SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
+SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
 create table hello.t (id int unsigned);
 create table hello.t (id int unsigned);
-select * from tb where data >= ''
-select * from tb where data >= ''
+select * from tb where data >= '';
+select * from tb where data >= '';
+alter table tb alter column id drop default;
+alter table tb alter column id drop default;

ast/testdata/TestFormat.golden

@@ -846,7 +846,7 @@ ADD
               ADD  
           index  `idx_store_film` (
             `store_id`, `film_id`);
-SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
 
 SELECT  
   DATE_FORMAT( t. atm, '%Y-%m-%d'
@@ -864,14 +864,20 @@ COUNT( DISTINCT  (
 ) 
 ORDER BY  
   DATE_FORMAT( t. atm, '%Y-%m-%d'
-)
+);
 create table hello.t (id int unsigned);
 create  table  hello. t  (id  int  unsigned);
-select * from tb where data >= ''
+select * from tb where data >= '';
 
 SELECT  
   * 
 FROM  
   tb  
 WHERE  
-  data  >= ''
+  data  >= '';
+alter table tb alter column id drop default;
+
+ALTER TABLE  
+  tb  alter  column  id  
+DROP  
+  DEFAULT;

ast/testdata/TestPretty.golden

@@ -1490,7 +1490,7 @@ ADD
               ADD  
           index  `idx_store_film` (
             `store_id`, `film_id`);
-SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
 
 SELECT  
   DATE_FORMAT( t. atm, '%Y-%m-%d'
@@ -1508,14 +1508,20 @@ COUNT( DISTINCT  (
 ) 
 ORDER BY  
   DATE_FORMAT( t. atm, '%Y-%m-%d'
-)
+);
 create table hello.t (id int unsigned);
 create  table  hello. t  (id  int  unsigned);
-select * from tb where data >= ''
+select * from tb where data >= '';
 
 SELECT  
   * 
 FROM  
   tb  
 WHERE  
-  data  >= ''
+  data  >= '';
+alter table tb alter column id drop default;
+
+ALTER TABLE  
+  tb  alter  column  id  
+DROP  
+  DEFAULT;

ast/testdata/TestTokenize.golden

@@ -160,9 +160,11 @@ alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
 [{5 ALTER TABLE  0} {1 inventory  0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 ; 0}]
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
 [{5 ALTER TABLE  0} {1 inventory  0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 , 0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 , 0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 ; 0}]
-SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
-[{5 SELECT	 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {7 , 0} {0   0} {4 COUNT( 0} {1 DISTINCT  0} {7 ( 0} {1 t. 0} {1 usr) 0} {7 ) 0} {0   0} {5 FROM  0} {1 usr_terminal  0} {1 t  0} {5 WHERE  0} {1 t. 0} {1 atm  0} {7 > 0} {0   0} {2 '2018-10-22 00:00:00' 0} {0   0} {6 AND  0} {1 t. 0} {1 agent  0} {1 LIKE  0} {2 '%Chrome%' 0} {0   0} {6 AND  0} {1 t. 0} {1 system  0} {7 = 0} {0   0} {2 'eip' 0} {0   0} {5 GROUP BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {0   0} {5 ORDER BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0}]
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');
+[{5 SELECT	 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {7 , 0} {0   0} {4 COUNT( 0} {1 DISTINCT  0} {7 ( 0} {1 t. 0} {1 usr) 0} {7 ) 0} {0   0} {5 FROM  0} {1 usr_terminal  0} {1 t  0} {5 WHERE  0} {1 t. 0} {1 atm  0} {7 > 0} {0   0} {2 '2018-10-22 00:00:00' 0} {0   0} {6 AND  0} {1 t. 0} {1 agent  0} {1 LIKE  0} {2 '%Chrome%' 0} {0   0} {6 AND  0} {1 t. 0} {1 system  0} {7 = 0} {0   0} {2 'eip' 0} {0   0} {5 GROUP BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {0   0} {5 ORDER BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {7 ; 0}]
 create table hello.t (id int unsigned);
 [{1 create  0} {1 table  0} {1 hello. 0} {1 t  0} {7 ( 0} {1 id  0} {1 int  0} {1 unsigned) 0} {7 ; 0}]
-select * from tb where data >= ''
-[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 tb  0} {5 WHERE  0} {1 data  0} {7 >= 0} {0   0} {2 '' 0}]
+select * from tb where data >= '';
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 tb  0} {5 WHERE  0} {1 data  0} {7 >= 0} {0   0} {2 '' 0} {7 ; 0}]
+alter table tb alter column id drop default;
+[{5 ALTER TABLE  0} {1 tb  0} {1 alter  0} {1 column  0} {1 id  0} {5 DROP  0} {4 DEFAULT; 0}]

cmd/soar/soar.go

@@ -187,8 +187,6 @@ func main() {
 			common.Log.Warning(errContent)
 			if common.Config.OnlySyntaxCheck {
 				fmt.Println(errContent)
-			}
-			if !common.Config.DryRun {
 				os.Exit(1)
 			}
 			// tidb parser 语法检查给出的建议 ERR.000

common/cases.go

@@ -196,11 +196,14 @@ func init() {
 		"alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);",
 
 		// https://github.com/XiaoMi/soar/issues/47
-		`SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')`,
+		`SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d');`,
 		// https://github.com/XiaoMi/soar/issues/17
 		"create table hello.t (id int unsigned);",
 
 		// https://github.com/XiaoMi/soar/issues/146
-		"select * from tb where data >= ''",
+		"select * from tb where data >= '';",
+
+		// https://github.com/XiaoMi/soar/issues/163
+		"alter table tb alter column id drop default;",
 	}
 }

common/config.go

@@ -110,6 +110,7 @@ type Configuration struct {
 	MaxSubqueryDepth     int      `yaml:"max-subquery-depth"`        // 子查询最大尝试
 	MaxVarcharLength     int      `yaml:"max-varchar-length"`        // varchar最大长度
 	ColumnNotAllowType   []string `yaml:"column-not-allow-type"`     // 字段不允许使用的数据类型
+	MinCardinality       float64  `yaml:"min-cardinality"`           // 添加索引散粒度阈值，范围 0~100
 
 	// ++++++++++++++EXPLAIN检查项+++++++++++++
 	ExplainSQLReportType   string   `yaml:"explain-sql-report-type"`  // EXPLAIN markdown 格式输出 SQL 样式，支持 sample, fingerprint, pretty 等
@@ -163,6 +164,7 @@ var Config = &Configuration{
 	ConnTimeOut:             3,
 	QueryTimeOut:            30,
 	Delimiter:               ";",
+	MinCardinality:          0,
 
 	MaxJoinTableCount:    5,
 	MaxGroupByColsCount:  5,
@@ -512,6 +514,7 @@ func readCmdFlags() error {
 	connTimeOut := flag.Int("conn-time-out", Config.ConnTimeOut, "ConnTimeOut, 数据库连接超时时间，单位秒")
 	queryTimeOut := flag.Int("query-time-out", Config.QueryTimeOut, "QueryTimeOut, 数据库SQL执行超时时间，单位秒")
 	delimiter := flag.String("delimiter", Config.Delimiter, "Delimiter, SQL分隔符")
+	minCardinality := flag.Float64("min-cardinality", Config.MinCardinality, "MinCardinality，索引列散粒度最低阈值，散粒度低于该值的列不添加索引，建议范围0.0 ~ 100.0")
 	// +++++++++++++++日志相关+++++++++++++++++
 	logLevel := flag.Int("log-level", Config.LogLevel, "LogLevel, 日志级别, [0:Emergency, 1:Alert, 2:Critical, 3:Error, 4:Warning, 5:Notice, 6:Informational, 7:Debug]")
 	logOutput := flag.String("log-output", Config.LogOutput, "LogOutput, 日志输出位置")
@@ -612,6 +615,7 @@ func readCmdFlags() error {
 	Config.IgnoreRules = strings.Split(*ignoreRules, ",")
 	Config.RewriteRules = strings.Split(*rewriteRules, ",")
 	*blackList = strings.TrimSpace(*blackList)
+	Config.MinCardinality = *minCardinality
 
 	if filepath.IsAbs(*blackList) || *blackList == "" {
 		Config.BlackList = *blackList

common/testdata/TestPrintConfiguration.golden

@@ -71,6 +71,7 @@ max-subquery-depth: 5
 max-varchar-length: 1024
 column-not-allow-type:
 - boolean
+min-cardinality: 0
 explain-sql-report-type: pretty
 explain-type: extended
 explain-format: traditional

deps.sh

@@ -10,3 +10,22 @@ for cmd in ${NEEDED_COMMANDS} ; do
         echo "${cmd} found"
     fi
 done
+
+# MySQL client
+## Mac OS: brew install mysql
+## Ubuntu: apt-get install mysql-client
+
+# Docker
+## https://www.docker.com
+
+# Git
+## https://git-scm.com/
+
+# Go
+## https://golang.org/
+
+# Govendor
+## go get github.com/kardianos/govendor
+
+# retool
+## go get github.com/twitchtv/retool

doc/example/main_test.sh

 #!/bin/bash
 
-
+GOPATH=$(go env GOPATH)
 PROJECT_PATH=${GOPATH}/src/github.com/XiaoMi/soar/
 
 if [ "$1x" == "-updatex" ]; then
-	cd "${PROJECT_PATH}" && ./bin/soar -list-test-sqls | ./bin/soar -config=../etc/soar.yaml > ./doc/example/main_test.md
+  cd "${PROJECT_PATH}" && ./bin/soar -list-test-sqls | ./bin/soar -config=../etc/soar.yaml > ./doc/example/main_test.md
+  if [ ! $? -eq 0 ]; then
+    exit 1
+  fi
 else
-	cd "${PROJECT_PATH}" && ./bin/soar -list-test-sqls | ./bin/soar -config=../etc/soar.yaml > ./doc/example/main_test.log
-    # optimizer_XXX 库名，散粒度，以及索引先后顺序每次可能会不一致
-    DIFF_LINES=$(cat ./doc/example/main_test.log ./doc/example/main_test.md | grep -v "optimizer\|散粒度" | sort | uniq -u | wc -l)
-    if [ "${DIFF_LINES}" -gt 0 ]; then
-    	git diff ./doc/example/main_test.log ./doc/example/main_test.md
-    fi
+  cd "${PROJECT_PATH}" && ./bin/soar -list-test-sqls | ./bin/soar -config=../etc/soar.yaml > ./doc/example/main_test.log
+  if [ ! $? -eq 0 ]; then
+    exit 1
+  fi
+  # optimizer_XXX 库名，散粒度，以及索引先后顺序每次可能会不一致
+  DIFF_LINES=$(cat ./doc/example/main_test.log ./doc/example/main_test.md | grep -v "optimizer\|散粒度" | sort | uniq -u | wc -l)
+  if [ "${DIFF_LINES}" -gt 0 ]; then
+    git diff ./doc/example/main_test.log ./doc/example/main_test.md
+  fi
 fi
-
-

doc/heuristic.md

@@ -532,6 +532,16 @@ CREATE TABLE tab (a varchar(3500));
 ```sql
 CREATE TABLE tab (a BOOLEAN);
 ```
+## 不建议使用精度在秒级以下的时间数据类型
+
+* **Item**:COL.019
+* **Severity**:L1
+* **Content**:使用高精度的时间数据类型带来的存储空间消耗相对较大；MySQL 在5.6.4以上才可以支持精确到微秒的时间数据类型，使用时需要考虑版本兼容问题。
+* **Case**:
+
+```sql
+CREATE TABLE t1 (t TIME(3), dt DATETIME(6));
+```
 ## 消除不必要的 DISTINCT 条件
 
 * **Item**:DIS.001

vendor/github.com/pingcap/parser/ast/dml.go

@@ -135,7 +135,18 @@ type TableName struct {
 
 // Restore implements Recoverable interface.
 func (n *TableName) Restore(sb *strings.Builder) error {
-	return errors.New("Not implemented")
+	if n.Schema.String() != "" {
+		WriteName(sb, n.Schema.String())
+		sb.WriteString(".")
+	}
+	WriteName(sb, n.Name.String())
+	for _, value := range n.IndexHints {
+		sb.WriteString(" ")
+		if err := value.Restore(sb); err != nil {
+			return errors.Annotate(err, "An error occurred while splicing IndexHints")
+		}
+	}
+	return nil
 }
 
 // IndexHintType is the type for index hint use, ignore or force.
@@ -166,6 +177,48 @@ type IndexHint struct {
 	HintScope  IndexHintScope
 }
 
+// IndexHint Restore (The const field uses switch to facilitate understanding)
+func (n *IndexHint) Restore(sb *strings.Builder) error {
+	indexHintType := ""
+	switch n.HintType {
+	case 1:
+		indexHintType = "USE INDEX"
+	case 2:
+		indexHintType = "IGNORE INDEX"
+	case 3:
+		indexHintType = "FORCE INDEX"
+	default: // Prevent accidents
+		return errors.New("IndexHintType has an error while matching")
+	}
+
+	indexHintScope := ""
+	switch n.HintScope {
+	case 1:
+		indexHintScope = ""
+	case 2:
+		indexHintScope = " FOR JOIN"
+	case 3:
+		indexHintScope = " FOR ORDER BY"
+	case 4:
+		indexHintScope = " FOR GROUP BY"
+	default: // Prevent accidents
+		return errors.New("IndexHintScope has an error while matching")
+	}
+
+	sb.WriteString(indexHintType)
+	sb.WriteString(indexHintScope)
+	sb.WriteString(" (")
+	for i, value := range n.IndexNames {
+		if i > 0 {
+			sb.WriteString(", ")
+		}
+		WriteName(sb, value.O)
+	}
+	sb.WriteString(")")
+
+	return nil
+}
+
 // Accept implements Node Accept interface.
 func (n *TableName) Accept(v Visitor) (Node, bool) {
 	newNode, skipChildren := v.Enter(n)

vendor/github.com/pingcap/parser/types/field_type.go

@@ -50,6 +50,12 @@ func NewFieldType(tp byte) *FieldType {
 	}
 }
 
+// Clone returns a copy of itself.
+func (ft *FieldType) Clone() *FieldType {
+	ret := *ft
+	return &ret
+}
+
 // Equal checks whether two FieldType objects are equal.
 func (ft *FieldType) Equal(other *FieldType) bool {
 	// We do not need to compare whole `ft.Flag == other.Flag` when wrapping cast upon an Expression.

vendor/github.com/pingcap/tidb/types/field_type.go

@@ -40,12 +40,6 @@ func NewFieldType(tp byte) *FieldType {
 	}
 }
 
-// CloneFieldType clones the given FieldType.
-func CloneFieldType(src *FieldType) *FieldType {
-	ft := *src
-	return &ft
-}
-
 // AggFieldType aggregates field types for a multi-argument function like `IF`, `IFNULL`, `COALESCE`
 // whose return type is determined by the arguments' FieldTypes.
 // Aggregation is performed by MergeFieldType function.