make SQL more compatible with pingcap/parser

.travis.yml

@@ -8,12 +8,7 @@ sudo: required
 
 go_import_path: github.com/XiaoMi/soar
 
-addons:
-  apt:
-   sources:
-    - mysql-5.7-trusty
-   packages:
-    - mysql-client
+dist: xenial
 
 services:
   - docker

ast/tidb.go

@@ -18,6 +18,8 @@ package ast
 
 import (
 	"fmt"
+	"regexp"
+	"strings"
 
 	"github.com/XiaoMi/soar/common"
 
@@ -34,6 +36,7 @@ import (
 // TiParse TiDB 语法解析
 func TiParse(sql, charset, collation string) ([]ast.StmtNode, error) {
 	p := parser.New()
+	sql = removeIncompatibleWords(sql)
 	stmt, warn, err := p.Parse(sql, charset, collation)
 	// TODO: bypass warning info
 	for _, w := range warn {
@@ -42,6 +45,37 @@ func TiParse(sql, charset, collation string) ([]ast.StmtNode, error) {
 	return stmt, err
 }
 
+// removeIncompatibleWords remove pingcap/parser not support words from schema
+func removeIncompatibleWords(sql string) string {
+	fields := strings.Fields(strings.TrimSpace(sql))
+	if len(fields) == 0 {
+		return sql
+	}
+	switch strings.ToLower(fields[0]) {
+	case "create", "alter":
+	default:
+		return sql
+	}
+	// CONSTRAINT col_fk FOREIGN KEY (col) REFERENCES tb (id) ON UPDATE CASCADE
+	re := regexp.MustCompile(`(?i) ON UPDATE CASCADE`)
+	sql = re.ReplaceAllString(sql, "")
+
+	// FULLTEXT KEY col_fk (col) /*!50100 WITH PARSER `ngram` */
+	// /*!50100 PARTITION BY LIST (col)
+	re = regexp.MustCompile(`/\*!5`)
+	sql = re.ReplaceAllString(sql, "/* 5")
+
+	// col varchar(10) CHARACTER SET gbk DEFAULT NULL
+	re = regexp.MustCompile(`(?i)CHARACTER SET [a-z_0-9]* `)
+	sql = re.ReplaceAllString(sql, "")
+
+	// CREATE TEMPORARY TABLE IF NOT EXISTS t_film AS (SELECT * FROM film);
+	re = regexp.MustCompile(`(?i)CREATE TEMPORARY TABLE`)
+	sql = re.ReplaceAllString(sql, "CREATE TABLE")
+
+	return sql
+}
+
 // PrintPrettyStmtNode 打印TiParse语法树
 func PrintPrettyStmtNode(sql, charset, collation string) {
 	tree, err := TiParse(sql, charset, collation)

ast/tidb_test.go

@@ -86,3 +86,31 @@ func TestSchemaMetaInfo(t *testing.T) {
 	}
 	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
 }
+
+func TestRemoveIncompatibleWords(t *testing.T) {
+	common.Log.Debug("Entering function: %s", common.GetFunctionName())
+	sqls := [][]string{
+		{
+			`CREATE TEMPORARY TABLE IF NOT EXISTS t_film AS (SELECT * FROM film)`,
+			`CREATE CONSTRAINT col_fk FOREIGN KEY (col) REFERENCES tb (id) ON UPDATE CASCADE`,
+			"CREATE FULLTEXT KEY col_fk (col) /*!50100 WITH PARSER `ngram` */",
+			`CREATE /*!50100 PARTITION BY LIST (col)`,
+			`CREATE col varchar(10) CHARACTER SET gbk DEFAULT NULL`,
+		},
+		{
+			`CREATE TABLE IF NOT EXISTS t_film AS (SELECT * FROM film)`,
+			`CREATE CONSTRAINT col_fk FOREIGN KEY (col) REFERENCES tb (id)`,
+			"CREATE FULLTEXT KEY col_fk (col) /* 50100 WITH PARSER `ngram` */",
+			`CREATE /* 50100 PARTITION BY LIST (col)`,
+			`CREATE col varchar(10) DEFAULT NULL`,
+		},
+	}
+	for k, sql := range sqls[0] {
+		sql = removeIncompatibleWords(sql)
+		if sqls[1][k] != sql {
+			fmt.Println(sql)
+			t.Fatal(sql)
+		}
+	}
+	common.Log.Debug("Exiting function: %s", common.GetFunctionName())
+}

test/fixture/test_Run_all_test_cases.golden

@@ -3861,7 +3861,7 @@ GROUP BY
 
 | id | select\_type | table | partitions | type | possible_keys | key | key\_len | ref | rows | filtered | scalability | Extra |
 |---|---|---|---|---|---|---|---|---|---|---|---|---|
-| 1  | SIMPLE | *actor* | NULL | ref | idx\_actor\_last\_name | idx\_actor\_last\_name | 137 | const | 2 | 33.33% | O(log n) | Using where; Using temporary |
+| 1  | SIMPLE | *actor* | NULL | ref | idx\_actor\_last\_name | idx\_actor\_last\_name | 137 | const | 2 | 16.67% | O(log n) | Using where; Using temporary |