提交 b93aad46 编写于 作者: martianzhang's avatar martianzhang

update pingcap/parser

	make cover passed
	make test-cli passed
上级 a5cb33b9
......@@ -1752,11 +1752,24 @@ func (q *Query4Audit) RuleUNIONLimit() Rule {
var rule = q.RuleOK()
for _, tiStmtNode := range q.TiStmt {
switch stmt := tiStmtNode.(type) {
case *tidb.UnionStmt:
// SetOprStmt represents "union/except/intersect statement"
case *tidb.SetOprStmt:
if stmt.Limit != nil {
for _, sel := range stmt.SelectList.Selects {
if sel.Limit == nil {
rule = HeuristicRules["SUB.007"]
switch n := sel.(type) {
case *tidb.SelectStmt:
if n.Limit == nil {
rule = HeuristicRules["SUB.007"]
}
case *tidb.SetOprSelectList:
for _, s := range n.Selects {
switch s1 := s.(type) {
case *tidb.SelectStmt:
if s1.Limit == nil {
rule = HeuristicRules["SUB.007"]
}
}
}
}
}
}
......@@ -1840,7 +1853,7 @@ func (q *Query4Audit) RuleImpreciseDataType() Rule {
continue
}
switch col.Tp.Tp {
case mysql.TypeFloat, mysql.TypeDouble, mysql.TypeDecimal, mysql.TypeNewDecimal:
case mysql.TypeFloat, mysql.TypeDouble, mysql.TypeNewDecimal:
rule = HeuristicRules["COL.009"]
}
}
......@@ -1855,8 +1868,7 @@ func (q *Query4Audit) RuleImpreciseDataType() Rule {
continue
}
switch col.Tp.Tp {
case mysql.TypeFloat, mysql.TypeDouble,
mysql.TypeDecimal, mysql.TypeNewDecimal:
case mysql.TypeFloat, mysql.TypeDouble, mysql.TypeNewDecimal:
rule = HeuristicRules["COL.009"]
}
}
......@@ -3152,7 +3164,9 @@ func (q *Query4Audit) RuleTimestampDefault() Rule {
hasDefault = true
if err := option.Restore(ctx); err == nil {
if strings.HasPrefix(strings.ToLower(sb.String()), `default '0`) ||
strings.HasPrefix(strings.ToLower(sb.String()), `default 0`) {
strings.HasPrefix(strings.ToLower(sb.String()), `default 0`) ||
strings.HasPrefix(strings.ToLower(sb.String()), `default _utf8mb4'0`) ||
strings.HasPrefix(strings.ToLower(sb.String()), `default _utf8'0`) {
hasDefault = false
}
}
......
......@@ -2430,6 +2430,7 @@ func TestRuleUNIONLimit(t *testing.T) {
},
{
`(SELECT * FROM tb1 ORDER BY name LIMIT 20) UNION ALL (SELECT * FROM tb2 ORDER BY name LIMIT 20) LIMIT 20;`,
`SELECT * FROM tb1 ORDER BY name LIMIT 20`,
},
}
for _, sql := range sqls[0] {
......@@ -2437,7 +2438,7 @@ func TestRuleUNIONLimit(t *testing.T) {
if err == nil {
rule := q.RuleUNIONLimit()
if rule.Item != "SUB.007" {
t.Error("Rule not match:", rule.Item, "Expect : SUB.007")
t.Error("Rule not match:", rule.Item, "Expect : SUB.007", sql)
}
} else {
t.Error("sqlparser.Parse Error:", err)
......@@ -3094,6 +3095,7 @@ func TestRuleTimestampDefault(t *testing.T) {
"CREATE TABLE tbl( `id` bigint not null, `create_time` timestamp) ENGINE=InnoDB DEFAULT CHARSET=utf8;",
"ALTER TABLE t1 MODIFY b timestamp NOT NULL;",
`ALTER TABLE t1 ADD c_time timestamp NOT NULL default "0000-00-00"`,
`ALTER TABLE t1 ADD c_time timestamp NOT NULL default '0'`,
`ALTER TABLE t1 ADD c_time timestamp NOT NULL default 0`,
`ALTER TABLE t1 ADD c_time datetime NOT NULL default 0`,
},
......@@ -3107,7 +3109,7 @@ func TestRuleTimestampDefault(t *testing.T) {
if err == nil {
rule := q.RuleTimestampDefault()
if rule.Item != "COL.013" {
t.Error("Rule not match:", rule.Item, "Expect : COL.013")
t.Error("Rule not match:", rule.Item, "Expect : COL.013", sql)
}
} else {
t.Error("sqlparser.Parse Error:", err)
......
......@@ -1723,8 +1723,8 @@ func MergeAlterTables(sqls ...string) map[string]string {
alterSQL = fmt.Sprint("ADD INDEX", " "+idxName+" ", buf)
case *ast.RenameTableStmt:
// 注意: 表名和库名不区分大小写
tableName = n.OldTable.Name.L
dbName = n.OldTable.Schema.L
tableName = n.TableToTables[0].OldTable.Name.L
dbName = n.TableToTables[0].OldTable.Schema.L
if alterExp.MatchString(sql) {
common.Log.Debug("rename alterExp: ALTER %v %v", tableName, alterExp.ReplaceAllString(sql, ""))
alterSQL = fmt.Sprint(alterExp.ReplaceAllString(sql, ""))
......
......@@ -2,10 +2,9 @@
&ast.SelectStmt{
dmlNode: ast.dmlNode{
stmtNode: ast.stmtNode{
node: ast.node{text:"select 1"},
node: ast.node{text:"select 1", offset:0},
},
},
resultSetNode: ast.resultSetNode{},
SelectStmtOpts: &ast.SelectStmtOpts{
Distinct: false,
SQLBigResult: false,
......@@ -16,6 +15,7 @@
StraightJoin: false,
Priority: 0,
TableHints: nil,
ExplicitAll: false,
},
Distinct: false,
From: (*ast.TableRefsClause)(nil),
......@@ -24,12 +24,12 @@
node: ast.node{},
Fields: {
&ast.SelectField{
node: ast.node{text:"1"},
node: ast.node{text:"1", offset:0},
Offset: 7,
WildCard: (*ast.WildCardField)(nil),
Expr: &driver.ValueExpr{
TexprNode: ast.exprNode{
node: ast.node{},
node: ast.node{text:"", offset:7},
Type: types.FieldType{
Tp: 0x8,
Flag: 0x80,
......@@ -43,10 +43,10 @@
},
Datum: types.Datum{
k: 0x1,
collation: "",
decimal: 0x0,
length: 0x0,
i: 1,
collation: "",
b: nil,
x: nil,
},
......@@ -57,16 +57,20 @@
},
},
},
GroupBy: (*ast.GroupByClause)(nil),
Having: (*ast.HavingClause)(nil),
WindowSpecs: nil,
OrderBy: (*ast.OrderByClause)(nil),
Limit: (*ast.Limit)(nil),
LockTp: 0,
TableHints: nil,
IsAfterUnionDistinct: false,
IsInBraces: false,
QueryBlockOffset: 0,
SelectIntoOpt: (*ast.SelectIntoOption)(nil),
GroupBy: (*ast.GroupByClause)(nil),
Having: (*ast.HavingClause)(nil),
WindowSpecs: nil,
OrderBy: (*ast.OrderByClause)(nil),
Limit: (*ast.Limit)(nil),
LockInfo: (*ast.SelectLockInfo)(nil),
TableHints: nil,
IsInBraces: false,
WithBeforeBraces: false,
QueryBlockOffset: 0,
SelectIntoOpt: (*ast.SelectIntoOption)(nil),
AfterSetOperator: (*ast.SetOprType)(nil),
Kind: 0x0,
Lists: nil,
With: (*ast.WithClause)(nil),
},
}
[
{
"text": "select 1",
"resultFields": null,
"offset": 0,
"SQLBigResult": false,
"SQLBufferResult": false,
"SQLCache": true,
......@@ -9,18 +9,22 @@
"CalcFoundRows": false,
"StraightJoin": false,
"Priority": 0,
"ExplicitAll": false,
"Distinct": false,
"From": null,
"Where": null,
"Fields": {
"text": "",
"offset": 0,
"Fields": [
{
"text": "1",
"offset": 0,
"Offset": 7,
"WildCard": null,
"Expr": {
"text": "",
"offset": 7,
"Type": {
"Tp": 8,
"Flag": 128,
......@@ -32,10 +36,10 @@
},
"flag": 0,
"k": 1,
"collation": "",
"decimal": 0,
"length": 0,
"i": 1,
"collation": "",
"b": null,
"x": null,
"projectionOffset": -1
......@@ -53,12 +57,16 @@
"WindowSpecs": null,
"OrderBy": null,
"Limit": null,
"LockTp": 0,
"LockInfo": null,
"TableHints": null,
"IsAfterUnionDistinct": false,
"IsInBraces": false,
"WithBeforeBraces": false,
"QueryBlockOffset": 0,
"SelectIntoOpt": null
"SelectIntoOpt": null,
"AfterSetOperator": null,
"Kind": 0,
"Lists": null,
"With": null
}
]
......@@ -126,7 +126,8 @@ func SchemaMetaInfo(sql string, defaultDatabase string) []string {
switch n := node.(type) {
case *ast.UseStmt:
tables = append(tables, fmt.Sprintf("`%s`.`dual`", n.DBName))
case *ast.InsertStmt, *ast.SelectStmt, *ast.UnionStmt, *ast.UpdateStmt, *ast.DeleteStmt:
// SetOprStmt represents "union/except/intersect statement"
case *ast.InsertStmt, *ast.SelectStmt, *ast.SetOprStmt, *ast.UpdateStmt, *ast.DeleteStmt:
// DML/DQL: INSERT, SELECT, UPDATE, DELETE
for _, tableRef := range common.JSONFind(jsonString, "TableRefs") {
for _, source := range common.JSONFind(tableRef, "Source") {
......
......@@ -205,6 +205,13 @@ func ColumnSort(colList []*Column) []*Column {
// GetDataTypeBase 获取dataType中的数据类型,忽略长度
func GetDataTypeBase(dataType string) string {
dataType = strings.TrimSpace(dataType)
if dataType == "" {
return dataType
}
// smallint unsigned, remove unsigned
dataType = strings.Fields(dataType)[0]
// int(10), remote (10)
if i := strings.Index(dataType, "("); i > 0 {
return dataType[0:i]
}
......@@ -256,7 +263,7 @@ func (col *Column) GetDataBytes(dbVersion int) int {
case "tinyint", "smallint", "mediumint",
"int", "integer", "bigint",
"double", "real", "float", "decimal",
"numeric", "bit":
"numeric", "bit", "smallint unsigned":
// numeric
return numericStorageReq(col.DataType)
......
......@@ -521,7 +521,8 @@ func (db *Connector) explainAbleSQL(sql string) (string, error) {
var isSelect bool
for _, st := range tiStmt {
switch st.(type) {
case *tidb.SelectStmt, *tidb.UnionStmt:
// SetOprStmt represents "union/except/intersect statement"
case *tidb.SelectStmt, *tidb.SetOprStmt:
isSelect = true
default:
isSelect = false
......
......@@ -183,19 +183,23 @@ func (db *Connector) ColumnCardinality(tb, col string) float64 {
return 0
}
// 如果是视图或表中无数据,rowTotal 都为0
// 视图不需要加索引,无数据相当于散粒度为1
// 如果是视图或表中无数据,rowTotal 都为 0
// 视图不需要加索引,无数据相当于散粒度为 1
if len(tbStatus.Rows) == 0 {
common.Log.Debug("(db *Connector) ColumnCardinality() No table status: %s", tb)
return 1
}
if tbStatus.Rows[0].Rows == nil {
common.Log.Debug("(db *Connector) ColumnCardinality() No table status: %s", tb)
return 1
}
rowTotal, err := strconv.ParseUint(string(tbStatus.Rows[0].Rows), 10, 64)
if rowTotal == 0 || err != nil {
if common.Config.Sampling {
common.Log.Debug("ColumnCardinality, %s rowTotal == 0", tb)
}
if err != nil {
common.Log.Error("ColumnCardinality, Error:", err.Error())
common.Log.Error("ColumnCardinality, ParseUint: " + string(tbStatus.Rows[0].Rows) + " Error: " + err.Error())
}
return 1
}
......@@ -244,9 +248,56 @@ func (db *Connector) IsView(tbName string) bool {
}
if len(tbStatus.Rows) > 0 {
if string(tbStatus.Rows[0].Comment) != "TABLE" {
/*
// mysql 8.0.23
mysql> show table status like "actor_info"\G
*************************** 1. row ***************************
Name: actor_info
Engine: NULL
Version: NULL
Row_format: NULL
Rows: NULL
Avg_row_length: NULL
Data_length: NULL
Max_data_length: NULL
Index_length: NULL
Data_free: NULL
Auto_increment: NULL
Create_time: 2021-06-01 10:56:47
Update_time: NULL
Check_time: NULL
Collation: NULL
Checksum: NULL
Create_options: NULL
Comment: VIEW
mysql> show table status like "film"\G
*************************** 1. row ***************************
Name: film
Engine: InnoDB
Version: 10
Row_format: Dynamic
Rows: 1000
Avg_row_length: 196
Data_length: 196608
Max_data_length: 0
Index_length: 81920
Data_free: 0
Auto_increment: 1001
Create_time: 2021-06-01 10:56:47
Update_time: NULL
Check_time: NULL
Collation: utf8_general_ci
Checksum: NULL
Create_options:
Comment:
*/
if string(tbStatus.Rows[0].Comment) == "VIEW" {
return true
}
if string(tbStatus.Rows[0].Comment) == "TABLE" {
return false
}
}
return false
......
......@@ -203,6 +203,10 @@ func TestIsView(t *testing.T) {
if !connTest.IsView("actor_info") {
t.Error("actor_info should be a VIEW")
}
if connTest.IsView("film") {
t.Error("film should be a TABLE")
}
connTest.Database = originalDatabase
common.Log.Debug("Exiting function: %s", common.GetFunctionName())
}
......
......@@ -15,7 +15,7 @@ require (
github.com/gedex/inflector v0.0.0-20170307190818-16278e9db813
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab // indirect
github.com/go-sql-driver/mysql v1.6.0
github.com/golang/snappy v0.0.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/gorilla/handlers v1.5.1 // indirect
github.com/gorilla/mux v1.8.0 // indirect
......@@ -25,7 +25,9 @@ require (
github.com/hashicorp/go-uuid v1.0.2 // indirect
github.com/hashicorp/serf v0.9.2 // indirect
github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c // indirect
github.com/jeremywohl/flatten v0.0.0-20190921043622-d936035e55cf // indirect
github.com/klauspost/pgzip v1.2.4 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
github.com/kr/pretty v0.2.1
github.com/looplab/fsm v0.2.0 // indirect
github.com/martini-contrib/auth v0.0.0-20150219114609-fa62c19b7ae8 // indirect
......@@ -39,19 +41,26 @@ require (
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/percona/go-mysql v0.0.0-20210427141028-73d29c6da78c
github.com/pingcap/parser v0.0.0-20200325032611-b7d1e7e1b93d
github.com/pingcap/tidb v1.1.0-beta.0.20200325081839-4a7d477399f4
github.com/pingcap/tipb v0.0.0-20210425040103-dc47a87b52aa // indirect
github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 // indirect
github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307
github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 // indirect
github.com/pingcap/tidb v1.1.0-beta.0.20210601085537-5d7c852770eb
github.com/pingcap/tipb v0.0.0-20210601083426-79a378b6d1c4 // indirect
github.com/planetscale/pargzip v0.0.0-20201116224723-90c7fc03ea8a // indirect
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
github.com/russross/blackfriday v1.6.0
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca
github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/sjmudd/stopwatch v0.0.0-20170613150411-f380bf8a9be1 // indirect
github.com/spf13/cobra v1.1.1 // indirect
github.com/spyzhov/ajson v0.4.2 // indirect
github.com/tidwall/gjson v1.7.5
golang.org/x/tools v0.0.0-20201202200335-bef1c476418a // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.17.0 // indirect
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b // indirect
golang.org/x/text v0.3.6 // indirect
gopkg.in/gcfg.v1 v1.2.3 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0
......
此差异已折叠。
{
"084DA3E3EE38DD85": [
"`unknown`.tb`"
"`unknown`.`tb`"
],
"08CFE41C7D20AAC8": [
"`unknown`.`city`",
......@@ -59,7 +59,7 @@
"`unknown`.`film`"
],
"255BAC03F56CDBC7": [
"`unknown`.address`"
"`unknown`.`address`"
],
"291F95B7DCB74C21": [
"`unknown`.`tb`"
......@@ -159,7 +159,7 @@
"`unknown`.`film`"
],
"9BB74D074BA0727C": [
"`unknown`.inventory`"
"`unknown`.`inventory`"
],
"A0C5E62C724A121A": [
"`sakila`.`film`"
......@@ -215,7 +215,7 @@
"`unknown`.`country`"
],
"C315BC4EE0F4E523": [
"`unknown`.inventory`"
"`unknown`.`inventory`"
],
"C3FAEDA6AD6D762B": [
"`unknown`.`film`"
......@@ -279,7 +279,7 @@
}
{
"084DA3E3EE38DD85": [
"`sakila`.tb`"
"`sakila`.`tb`"
],
"08CFE41C7D20AAC8": [
"`sakila`.`city`",
......@@ -338,7 +338,7 @@
"`sakila`.`film`"
],
"255BAC03F56CDBC7": [
"`sakila`.address`"
"`sakila`.`address`"
],
"291F95B7DCB74C21": [
"`sakila`.`tb`"
......@@ -438,7 +438,7 @@
"`sakila`.`film`"
],
"9BB74D074BA0727C": [
"`sakila`.inventory`"
"`sakila`.`inventory`"
],
"A0C5E62C724A121A": [
"`sakila`.`film`"
......@@ -494,7 +494,7 @@
"`sakila`.`country`"
],
"C315BC4EE0F4E523": [
"`sakila`.inventory`"
"`sakila`.`inventory`"
],
"C3FAEDA6AD6D762B": [
"`sakila`.`film`"
......
......@@ -4082,7 +4082,7 @@ FROM
# Query: E48A20D0413512DA
☆ ☆ ☆ ☆ 30分
★ ☆ ☆ ☆ 40分
```sql
......@@ -4112,7 +4112,7 @@ ORDER BY
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 1 | PRIMARY | *country* | NULL | index | PRIMARY | PRIMARY | 2 | NULL | 109 | ☠️ **100.00%** | ☠️ **O(n)** | Using index; Using temporary; Using filesort |
| 1 | PRIMARY | *city* | NULL | ref | PRIMARY,<br>idx\_fk\_country\_id | idx\_fk\_country\_id | 2 | sakila.country.country\_id | 5 | ☠️ **100.00%** | O(log n) | NULL |
| 1 | PRIMARY | *c* | NULL | ALL | NULL | NULL | NULL | NULL | 600 | n% | ☠️ **O(n)** | Using where; Using join buffer (Block Nested Loop) |
| 1 | PRIMARY | *c* | NULL | ALL | NULL | NULL | NULL | NULL | 600 | n% | ☠️ **O(n)** | Using where; Using join buffer (hash join) |
| 1 | PRIMARY | *a* | NULL | ref | PRIMARY,<br>idx\_fk\_city\_id | idx\_fk\_city\_id | 2 | sakila.city.city\_id | 1 | ☠️ **100.00%** | O(log n) | NULL |
| 1 | PRIMARY | *cu* | NULL | ref | idx\_fk\_address\_id | idx\_fk\_address\_id | 2 | sakila.a.address\_id | 1 | ☠️ **100.00%** | O(log n) | NULL |
| 1 | PRIMARY | *<derived2>* | NULL | ref | <auto\_key0> | <auto\_key0> | 152 | sakila.a.address | 6 | ☠️ **100.00%** | O(log n) | Using index |
......@@ -4166,18 +4166,6 @@ ORDER BY
## 为sakila库的customer\_list表添加索引
* **Item:** IDX.002
* **Severity:** L2
* **Content:** 为列city添加索引,散粒度为: n%; 为列address添加索引,散粒度为: n%; 为列SID添加索引,散粒度为: n%; 为列phone添加索引,散粒度为: n%;
* **Case:** ALTER TABLE \`sakila\`.\`customer\_list\` add index \`idx\_city\` (\`city\`), add index \`idx\_address\` (\`address\`), add index \`idx\_SID\_phone\` (\`SID\`,\`phone\`) ;
## 建议使用 AS 关键字显示声明一个别名
* **Item:** ALI.001
......
......@@ -5,7 +5,7 @@ online-dsn:
addr: 127.0.0.1:3306
schema: information_schema
charset: utf8
collation: utf8_general_ci
collation: utf8mb4_general_ci
loc: UTC
tls: ""
server-public-key: ""
......@@ -25,7 +25,7 @@ test-dsn:
addr: 127.0.0.1:3306
schema: information_schema
charset: utf8
collation: utf8_general_ci
collation: utf8mb4_general_ci
loc: UTC
tls: ""
server-public-key: ""
......
......@@ -103,7 +103,7 @@ load test_helper
# 14. soar -help 检查
@test "Check soar help" {
run ${SOAR_BIN} -help
[ $status -eq 2 ]
[ $status -eq 0 ]
[ "${#lines[@]}" -gt 30 ]
}
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册