fix #155

a new config arg 'min-cardinality' for index column selection

fix #155
a new config arg 'min-cardinality' for index column selection
87dbcaf3 · liipx · 1d52539d · 87dbcaf3 · 87dbcaf3 · 87dbcaf3
5 changed file
--- a/advisor/index.go
+++ b/advisor/index.go
@@ -628,7 +628,6 @@ func (idxAdv *IndexAdvisor) buildJoinIndex(meta common.Meta) []IndexInfo {
 		indexColsList := make(map[string]map[string][]*common.Column)
 		for _, col := range IndexCols {
 			mergeIndex(indexColsList, col)
-
 		}

 		if common.Config.TestDSN.Disable || common.Config.OnlineDSN.Disable {
@@ -723,6 +722,11 @@ func (idxAdv *IndexAdvisor) buildIndexWithNoEnv(indexList map[string]map[string]

 // mergeIndex 将索引用到的列去重后合并到一起
 func mergeIndex(idxList map[string]map[string][]*common.Column, column *common.Column) {
+	// 散粒度低于阈值将不会添加索引
+	if common.Config.MinCardinality > column.Cardinality {
+		return
+	}
+
 	db := column.DB
 	tb := column.Table
 	if idxList[db] == nil {

--- a/advisor/index_test.go
+++ b/advisor/index_test.go
@@ -357,6 +357,7 @@ func TestRuleUpdatePrimaryKey(t *testing.T) {

 func TestIndexAdvise(t *testing.T) {
 	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	// common.Config.MinCardinality = 1
 	vEnv, rEnv := env.BuildEnv()
 	defer vEnv.CleanUp()

@@ -377,7 +378,7 @@ func TestIndexAdvise(t *testing.T) {
 			if idxAdvisor != nil {
 				rule := idxAdvisor.IndexAdvise().Format()
 				if len(rule) > 0 {
-					pretty.Println(rule)
+					_, _ = pretty.Println(rule)
 				}
 			}
 		}

--- a/common/config.go
+++ b/common/config.go
@@ -110,6 +110,7 @@ type Configuration struct {
 	MaxSubqueryDepth     int      `yaml:"max-subquery-depth"`        // 子查询最大尝试
 	MaxVarcharLength     int      `yaml:"max-varchar-length"`        // varchar最大长度
 	ColumnNotAllowType   []string `yaml:"column-not-allow-type"`     // 字段不允许使用的数据类型
+	MinCardinality       float64  `yaml:"min-cardinality"`           // 添加索引散粒度阈值，范围 0~100

 	// ++++++++++++++EXPLAIN检查项+++++++++++++
 	ExplainSQLReportType   string   `yaml:"explain-sql-report-type"`  // EXPLAIN markdown 格式输出 SQL 样式，支持 sample, fingerprint, pretty 等
@@ -163,6 +164,7 @@ var Config = &Configuration{
 	ConnTimeOut:             3,
 	QueryTimeOut:            30,
 	Delimiter:               ";",
+	MinCardinality:          0,

 	MaxJoinTableCount:    5,
 	MaxGroupByColsCount:  5,
@@ -512,6 +514,7 @@ func readCmdFlags() error {
 	connTimeOut := flag.Int("conn-time-out", Config.ConnTimeOut, "ConnTimeOut, 数据库连接超时时间，单位秒")
 	queryTimeOut := flag.Int("query-time-out", Config.QueryTimeOut, "QueryTimeOut, 数据库SQL执行超时时间，单位秒")
 	delimiter := flag.String("delimiter", Config.Delimiter, "Delimiter, SQL分隔符")
+	minCardinality := flag.Float64("min-cardinality", Config.MinCardinality, "MinCardinality，索引列散粒度最低阈值，散粒度低于该值的列不添加索引，建议范围0.0 ~ 100.0")
 	// +++++++++++++++日志相关+++++++++++++++++
 	logLevel := flag.Int("log-level", Config.LogLevel, "LogLevel, 日志级别, [0:Emergency, 1:Alert, 2:Critical, 3:Error, 4:Warning, 5:Notice, 6:Informational, 7:Debug]")
 	logOutput := flag.String("log-output", Config.LogOutput, "LogOutput, 日志输出位置")
@@ -612,6 +615,7 @@ func readCmdFlags() error {
 	Config.IgnoreRules = strings.Split(*ignoreRules, ",")
 	Config.RewriteRules = strings.Split(*rewriteRules, ",")
 	*blackList = strings.TrimSpace(*blackList)
+	Config.MinCardinality = *minCardinality

 	if filepath.IsAbs(*blackList) || *blackList == "" {
 		Config.BlackList = *blackList

--- a/common/testdata/TestPrintConfiguration.golden
+++ b/common/testdata/TestPrintConfiguration.golden
@@ -71,6 +71,7 @@ max-subquery-depth: 5
 max-varchar-length: 1024
 column-not-allow-type:
 - boolean
+min-cardinality: 0
 explain-sql-report-type: pretty
 explain-type: extended
 explain-format: traditional

--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -167,44 +167,44 @@
 		{
 			"checksumSHA1": "fWqL/7jTYOiqDNmiUcQi3u45Hw0=",
 			"path": "github.com/pingcap/tidb/sessionctx/stmtctx",
-			"revision": "9f51a626e6587118db4306882a5578e7ffe3d2a6",
-			"revisionTime": "2018-12-13T05:17:24Z"
+			"revision": "4ccd7456fcdc5e62e68f49a0a542cbcf88e38343",
+			"revisionTime": "2018-12-13T09:52:39Z"
 		},
 		{
 			"checksumSHA1": "0CCq+3fAyaXs9XU+xWaRvbbtSOQ=",
 			"path": "github.com/pingcap/tidb/types",
-			"revision": "9f51a626e6587118db4306882a5578e7ffe3d2a6",
-			"revisionTime": "2018-12-13T05:17:24Z"
+			"revision": "4ccd7456fcdc5e62e68f49a0a542cbcf88e38343",
+			"revisionTime": "2018-12-13T09:52:39Z"
 		},
 		{
 			"checksumSHA1": "DWVD7+ygtT66IQ+cqXmMJ5OVqUk=",
 			"path": "github.com/pingcap/tidb/types/json",
-			"revision": "9f51a626e6587118db4306882a5578e7ffe3d2a6",
-			"revisionTime": "2018-12-13T05:17:24Z"
+			"revision": "4ccd7456fcdc5e62e68f49a0a542cbcf88e38343",
+			"revisionTime": "2018-12-13T09:52:39Z"
 		},
 		{
 			"checksumSHA1": "Zp5ME8OXNTmHnYTwJJUZlydN4/U=",
 			"path": "github.com/pingcap/tidb/types/parser_driver",
-			"revision": "9f51a626e6587118db4306882a5578e7ffe3d2a6",
-			"revisionTime": "2018-12-13T05:17:24Z"
+			"revision": "4ccd7456fcdc5e62e68f49a0a542cbcf88e38343",
+			"revisionTime": "2018-12-13T09:52:39Z"
 		},
 		{
 			"checksumSHA1": "s709bhSrG2Ec35406mGtrySid4s=",
 			"path": "github.com/pingcap/tidb/util/execdetails",
-			"revision": "9f51a626e6587118db4306882a5578e7ffe3d2a6",
-			"revisionTime": "2018-12-13T05:17:24Z"
+			"revision": "4ccd7456fcdc5e62e68f49a0a542cbcf88e38343",
+			"revisionTime": "2018-12-13T09:52:39Z"
 		},
 		{
 			"checksumSHA1": "nUC7zVoAMNR2a+z2iGqHoN2AkFE=",
 			"path": "github.com/pingcap/tidb/util/hack",
-			"revision": "9f51a626e6587118db4306882a5578e7ffe3d2a6",
-			"revisionTime": "2018-12-13T05:17:24Z"
+			"revision": "4ccd7456fcdc5e62e68f49a0a542cbcf88e38343",
+			"revisionTime": "2018-12-13T09:52:39Z"
 		},
 		{
 			"checksumSHA1": "xSyepiuqsoaaeDch7cXeumvVHKM=",
 			"path": "github.com/pingcap/tidb/util/memory",
-			"revision": "9f51a626e6587118db4306882a5578e7ffe3d2a6",
-			"revisionTime": "2018-12-13T05:17:24Z"
+			"revision": "4ccd7456fcdc5e62e68f49a0a542cbcf88e38343",
+			"revisionTime": "2018-12-13T09:52:39Z"
 		},
 		{
 			"checksumSHA1": "SmYeIK/fIYXNu8IKxD6HOVQVTuU=",
@@ -401,62 +401,62 @@
 		{
 			"checksumSHA1": "aKn1oKcY74N8TRLm3Ayt7Q4bbI4=",
 			"path": "vitess.io/vitess/go/bytes2",
-			"revision": "84a3524002899f761096c454e02924774c61d6ee",
-			"revisionTime": "2018-12-12T17:00:54Z"
+			"revision": "eb5a5e0e435db30856600b794d8cff235fdf24b7",
+			"revisionTime": "2018-12-13T22:28:01Z"
 		},
 		{
 			"checksumSHA1": "JVCEN4UGRmg3TofIBdzZMZ3G0Ww=",
 			"path": "vitess.io/vitess/go/hack",
-			"revision": "84a3524002899f761096c454e02924774c61d6ee",
-			"revisionTime": "2018-12-12T17:00:54Z"
+			"revision": "eb5a5e0e435db30856600b794d8cff235fdf24b7",
+			"revisionTime": "2018-12-13T22:28:01Z"
 		},
 		{
 			"checksumSHA1": "e1WJ7vCnVrlQQQlc6n/FewCDMso=",
 			"path": "vitess.io/vitess/go/sqltypes",
-			"revision": "84a3524002899f761096c454e02924774c61d6ee",
-			"revisionTime": "2018-12-12T17:00:54Z"
+			"revision": "eb5a5e0e435db30856600b794d8cff235fdf24b7",
+			"revisionTime": "2018-12-13T22:28:01Z"
 		},
 		{
 			"checksumSHA1": "ntFIQYkBS51G6y+FEkjFW40+HOU=",
 			"path": "vitess.io/vitess/go/vt/log",
-			"revision": "84a3524002899f761096c454e02924774c61d6ee",
-			"revisionTime": "2018-12-12T17:00:54Z"
+			"revision": "eb5a5e0e435db30856600b794d8cff235fdf24b7",
+			"revisionTime": "2018-12-13T22:28:01Z"
 		},
 		{
 			"checksumSHA1": "tPQFPwbMdjuX0qjNl4Zl8zc37JQ=",
 			"path": "vitess.io/vitess/go/vt/proto/query",
-			"revision": "84a3524002899f761096c454e02924774c61d6ee",
-			"revisionTime": "2018-12-12T17:00:54Z"
+			"revision": "eb5a5e0e435db30856600b794d8cff235fdf24b7",
+			"revisionTime": "2018-12-13T22:28:01Z"
 		},
 		{
 			"checksumSHA1": "o0tR/c7lgr0pLkxk7CdvjiNDAKU=",
 			"path": "vitess.io/vitess/go/vt/proto/topodata",
-			"revision": "84a3524002899f761096c454e02924774c61d6ee",
-			"revisionTime": "2018-12-12T17:00:54Z"
+			"revision": "eb5a5e0e435db30856600b794d8cff235fdf24b7",
+			"revisionTime": "2018-12-13T22:28:01Z"
 		},
 		{
 			"checksumSHA1": "77UojBqi0yyeQvR70j7C3kcKclQ=",
 			"path": "vitess.io/vitess/go/vt/proto/vtgate",
-			"revision": "84a3524002899f761096c454e02924774c61d6ee",
-			"revisionTime": "2018-12-12T17:00:54Z"
+			"revision": "eb5a5e0e435db30856600b794d8cff235fdf24b7",
+			"revisionTime": "2018-12-13T22:28:01Z"
 		},
 		{
 			"checksumSHA1": "QpWGhoVDwM+8+sgYLI/YU+95iGU=",
 			"path": "vitess.io/vitess/go/vt/proto/vtrpc",
-			"revision": "84a3524002899f761096c454e02924774c61d6ee",
-			"revisionTime": "2018-12-12T17:00:54Z"
+			"revision": "eb5a5e0e435db30856600b794d8cff235fdf24b7",
+			"revisionTime": "2018-12-13T22:28:01Z"
 		},
 		{
 			"checksumSHA1": "lENrUY/YyxwYFHYN+21TBH92P3U=",
 			"path": "vitess.io/vitess/go/vt/sqlparser",
-			"revision": "84a3524002899f761096c454e02924774c61d6ee",
-			"revisionTime": "2018-12-12T17:00:54Z"
+			"revision": "eb5a5e0e435db30856600b794d8cff235fdf24b7",
+			"revisionTime": "2018-12-13T22:28:01Z"
 		},
 		{
 			"checksumSHA1": "oF4XzuOzwvj1iduX/lYqNSyY/HM=",
 			"path": "vitess.io/vitess/go/vt/vterrors",
-			"revision": "84a3524002899f761096c454e02924774c61d6ee",
-			"revisionTime": "2018-12-12T17:00:54Z"
+			"revision": "eb5a5e0e435db30856600b794d8cff235fdf24b7",
+			"revisionTime": "2018-12-13T22:28:01Z"
 		}
 	],
 	"rootPath": "github.com/XiaoMi/soar"