提交 cb33d1fc 编写于 作者: martianzhang's avatar martianzhang

add new report-type `tables`

  -report-type tables will give SQL affect tables info
上级 bbd5e950
SELECT * FROM film WHERE length = 86;
[`sakila`.`film`]
SELECT * FROM film WHERE length IS NULL;
[`sakila`.`film`]
SELECT * FROM film HAVING title = 'abc';
[`sakila`.`film`]
SELECT * FROM sakila.film WHERE length >= 60;
[`sakila`.`film`]
SELECT * FROM sakila.film WHERE length >= '60';
[`sakila`.`film`]
SELECT * FROM film WHERE length BETWEEN 60 AND 84;
[`sakila`.`film`]
SELECT * FROM film WHERE title LIKE 'AIR%';
[`sakila`.`film`]
SELECT * FROM film WHERE title IS NOT NULL;
[`sakila`.`film`]
SELECT * FROM film WHERE length = 114 and title = 'ALABAMA DEVIL';
[`sakila`.`film`]
SELECT * FROM film WHERE length > 100 and title = 'ALABAMA DEVIL';
[`sakila`.`film`]
SELECT * FROM film WHERE length > 100 and language_id < 10 and title = 'xyz';
[`sakila`.`film`]
SELECT * FROM film WHERE length > 100 and language_id < 10;
[`sakila`.`film`]
SELECT release_year, sum(length) FROM film WHERE length = 123 AND language_id = 1 GROUP BY release_year;
[`sakila`.`film`]
SELECT release_year, sum(length) FROM film WHERE length >= 123 GROUP BY release_year;
[`sakila`.`film`]
SELECT release_year, language_id, sum(length) FROM film GROUP BY release_year, language_id;
[`sakila`.`film`]
SELECT release_year, sum(length) FROM film WHERE length = 123 GROUP BY release_year,(length+language_id);
[`sakila`.`film`]
SELECT release_year, sum(film_id) FROM film GROUP BY release_year;
[`sakila`.`film`]
SELECT * FROM address GROUP BY address,district;
[`sakila`.`address`]
SELECT title FROM film WHERE ABS(language_id) = 3 GROUP BY title;
[`sakila`.`film`]
SELECT language_id FROM film WHERE length = 123 GROUP BY release_year ORDER BY language_id;
[`sakila`.`film`]
SELECT release_year FROM film WHERE length = 123 GROUP BY release_year ORDER BY release_year;
[`sakila`.`film`]
SELECT * FROM film WHERE length = 123 ORDER BY release_year ASC, language_id DESC;
[`sakila`.`film`]
SELECT release_year FROM film WHERE length = 123 GROUP BY release_year ORDER BY release_year LIMIT 10;
[`sakila`.`film`]
SELECT * FROM film WHERE length = 123 ORDER BY release_year LIMIT 10;
[`sakila`.`film`]
SELECT * FROM film ORDER BY release_year LIMIT 10;
[`sakila`.`film`]
SELECT film_id FROM film ORDER BY release_year LIMIT 10;
[`sakila`.`film`]
SELECT * FROM film WHERE length > 100 ORDER BY length LIMIT 10;
[`sakila`.`film`]
SELECT * FROM film WHERE length < 100 ORDER BY length LIMIT 10;
[`sakila`.`film`]
SELECT * FROM customer WHERE address_id in (224,510) ORDER BY last_name;
[`sakila`.`customer`]
SELECT * FROM film WHERE release_year = 2016 AND length != 1 ORDER BY title;
[`sakila`.`film`]
SELECT title FROM film WHERE release_year = 1995;
[`sakila`.`film`]
SELECT title, replacement_cost FROM film WHERE language_id = 5 AND length = 70;
[`sakila`.`film`]
SELECT title FROM film WHERE language_id > 5 AND length > 70;
[`sakila`.`film`]
SELECT * FROM film WHERE length = 100 and title = 'xyz' ORDER BY release_year;
[`sakila`.`film`]
SELECT * FROM film WHERE length > 100 and title = 'xyz' ORDER BY release_year;
[`sakila`.`film`]
SELECT * FROM film WHERE length > 100 ORDER BY release_year;
[`sakila`.`film`]
SELECT * FROM city a INNER JOIN country b ON a.country_id=b.country_id;
[`sakila`.`country` `sakila`.`city`]
SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id;
[`sakila`.`city` `sakila`.`country`]
SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id;
[`sakila`.`city` `sakila`.`country`]
SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id WHERE b.last_update IS NULL;
[`sakila`.`city` `sakila`.`country`]
SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id WHERE a.last_update IS NULL;
[`sakila`.`country` `sakila`.`city`]
SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id UNION SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id;
[`sakila`.`country` `sakila`.`city`]
SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id WHERE a.last_update IS NULL UNION SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id WHERE b.last_update IS NULL;
[`sakila`.`city` `sakila`.`country`]
SELECT country_id, last_update FROM city NATURAL JOIN country;
[`sakila`.`country` `sakila`.`city`]
SELECT country_id, last_update FROM city NATURAL LEFT JOIN country;
[`sakila`.`city` `sakila`.`country`]
SELECT country_id, last_update FROM city NATURAL RIGHT JOIN country;
[`sakila`.`country` `sakila`.`city`]
SELECT a.country_id, a.last_update FROM city a STRAIGHT_JOIN country b ON a.country_id=b.country_id;
[`sakila`.`city` `sakila`.`country`]
SELECT a.address, a.postal_code FROM sakila.address a WHERE a.city_id IN (SELECT c.city_id FROM sakila.city c);
[`sakila`.`address` `sakila`.`city`]
SELECT city FROM( SELECT city_id FROM city WHERE city = "A Corua (La Corua)" ORDER BY last_update DESC LIMIT 50, 10) I JOIN city ON (I.city_id = city.city_id) JOIN country ON (country.country_id = city.country_id) ORDER BY city DESC;
[`sakila`.`city` `sakila`.`country`]
DELETE city, country FROM city INNER JOIN country using (country_id) WHERE city.city_id = 1;
[`sakila`.`country` `sakila`.`city`]
DELETE city FROM city LEFT JOIN country ON city.country_id = country.country_id WHERE country.country IS NULL;
[`sakila`.`city` `sakila`.`country`]
DELETE a1, a2 FROM city AS a1 INNER JOIN country AS a2 WHERE a1.country_id=a2.country_id;
[`sakila`.`city` `sakila`.`country`]
DELETE FROM a1, a2 USING city AS a1 INNER JOIN country AS a2 WHERE a1.country_id=a2.country_id;
[`sakila`.`city` `sakila`.`country`]
DELETE FROM film WHERE length > 100;
[`sakila`.`film`]
UPDATE city INNER JOIN country USING(country_id) SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.city_id=10;
[`sakila`.`country` `sakila`.`city`]
UPDATE city INNER JOIN country ON city.country_id = country.country_id INNER JOIN address ON city.city_id = address.city_id SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.city_id=10;
[`sakila`.`address`]
UPDATE city, country SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.country_id = country.country_id AND city.city_id=10;
[`sakila`.`country`]
UPDATE film SET length = 10 WHERE language_id = 20;
[`sakila`.`film`]
INSERT INTO city (country_id) SELECT country_id FROM country;
[`sakila`.`city` `sakila`.`country`]
INSERT INTO city (country_id) VALUES (1),(2),(3);
[`sakila`.`city`]
INSERT INTO city (country_id) VALUES (10);
[`sakila`.`city`]
INSERT INTO city (country_id) SELECT 10 FROM DUAL;
[`sakila`.`city`]
REPLACE INTO city (country_id) SELECT country_id FROM country;
[`sakila`.`city` `sakila`.`country`]
REPLACE INTO city (country_id) VALUES (1),(2),(3);
[`sakila`.`city`]
REPLACE INTO city (country_id) VALUES (10);
[`sakila`.`city`]
REPLACE INTO city (country_id) SELECT 10 FROM DUAL;
[`sakila`.`city`]
SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film;
[`sakila`.`film`]
SELECT * FROM film WHERE language_id = (SELECT language_id FROM language LIMIT 1);
[`sakila`.`film` `sakila`.`language`]
SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id;
[`sakila`.`city` `sakila`.`country`]
SELECT * FROM (SELECT * FROM actor WHERE last_update='2006-02-15 04:34:33' and last_name='CHASE') t WHERE last_update='2006-02-15 04:34:33' and last_name='CHASE' GROUP BY first_name;
[`sakila`.`actor`]
SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id;
[`sakila`.`city` `sakila`.`country`]
SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id WHERE o.country_id is null union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id WHERE i.city_id is null;
[`sakila`.`city` `sakila`.`country`]
SELECT first_name,last_name,email FROM customer STRAIGHT_JOIN address ON customer.address_id=address.address_id;
[`sakila`.`customer` `sakila`.`address`]
SELECT ID,name FROM (SELECT address FROM customer_list WHERE SID=1 order by phone limit 50,10) a JOIN customer_list l ON (a.address=l.address) JOIN city c ON (c.city=l.city) order by phone desc;
[`sakila`.`city` `sakila`.`customer_list`]
SELECT * FROM film WHERE date(last_update)='2006-02-15';
[`sakila`.`film`]
SELECT last_update FROM film GROUP BY date(last_update);
[`sakila`.`film`]
SELECT last_update FROM film order by date(last_update);
[`sakila`.`film`]
SELECT description FROM film WHERE description IN('NEWS','asd') GROUP BY description;
[`sakila`.`film`]
alter table address add index idx_city_id(city_id);
[`sakila`.address`]
alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
[`sakila`.inventory`]
alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
[`sakila`.inventory`]
SELECT DATE_FORMAT(t.last_update, '%Y-%m-%d'), COUNT(DISTINCT (t.city)) FROM city t WHERE t.last_update > '2018-10-22 00:00:00' AND t.city LIKE '%Chrome%' AND t.city = 'eip' GROUP BY DATE_FORMAT(t.last_update, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.last_update, '%Y-%m-%d');
[`sakila`.`city`]
create table hello.t (id int unsigned);
[`hello`.`t`]
select * from tb where data >= '';
[`sakila`.`tb`]
alter table tb alter column id drop default;
[`sakila`.tb`]
......@@ -17,13 +17,15 @@
package ast
import (
"fmt"
"github.com/XiaoMi/soar/common"
json "github.com/CorgiMan/json2"
"github.com/kr/pretty"
"github.com/pingcap/parser"
"github.com/pingcap/parser/ast"
json "github.com/CorgiMan/json2"
"github.com/tidwall/gjson"
// for pingcap parser
_ "github.com/pingcap/tidb/types/parser_driver"
......@@ -67,3 +69,69 @@ func StmtNode2JSON(sql, charset, collation string) string {
}
return str
}
// SchemaMetaInfo get used database, table name from SQL
func SchemaMetaInfo(sql string, defaultDatabase string) []string {
var tables []string
tree, err := TiParse(sql, "", "")
if err != nil {
return tables
}
jsonString := StmtNode2JSON(sql, "", "")
for _, node := range tree {
switch n := node.(type) {
case *ast.UseStmt:
tables = append(tables, fmt.Sprintf("`%s`.`dual`", n.DBName))
case *ast.InsertStmt, *ast.SelectStmt, *ast.UnionStmt, *ast.UpdateStmt, *ast.DeleteStmt:
// DML/DQL: INSERT, SELECT, UPDATE, DELETE
tableRefs := common.JSONFind(jsonString, "TableRefs")
for _, table := range tableRefs {
leftDatabase := gjson.Get(table, "Left.Source.Schema.O")
leftTable := gjson.Get(table, "Left.Source.Name.O")
if leftDatabase.String() == "" {
if leftTable.String() != "" {
tables = append(tables, fmt.Sprintf("`%s`.`%s`", defaultDatabase, leftTable))
}
} else {
if leftTable.String() != "" {
tables = append(tables, fmt.Sprintf("`%s`.`%s`", leftDatabase, leftTable))
} else {
tables = append(tables, fmt.Sprintf("`%s`.`dual`", leftDatabase))
}
}
rightDatabase := gjson.Get(table, "Right.Source.Schema.O")
rightTable := gjson.Get(table, "Right.Source.Name.O")
if rightDatabase.String() == "" {
if rightTable.String() != "" {
tables = append(tables, fmt.Sprintf("`%s`.`%s`", defaultDatabase, rightTable))
}
} else {
if rightTable.String() != "" {
tables = append(tables, fmt.Sprintf("`%s`.`%s`", rightDatabase, rightTable))
} else {
tables = append(tables, fmt.Sprintf("`%s`.`dual`", rightDatabase))
}
}
}
default:
// DDL: CREATE TABLE|DATABASE|INDEX|VIEW
schemas := common.JSONFind(jsonString, "Table")
for _, table := range schemas {
db := gjson.Get(table, "Schema.O")
tb := gjson.Get(table, "Name.O")
if db.String() == "" {
if tb.String() != "" {
tables = append(tables, fmt.Sprintf("`%s`.%s`", defaultDatabase, tb))
}
} else {
if tb.String() != "" {
tables = append(tables, fmt.Sprintf("`%s`.`%s`", db, tb))
}
}
}
}
}
return common.RemoveDuplicatesItem(tables)
}
......@@ -56,3 +56,17 @@ func TestStmtNode2JSON(t *testing.T) {
}
common.Log.Debug("Exiting function: %s", common.GetFunctionName())
}
func TestSchemaMetaInfo(t *testing.T) {
common.Log.Debug("Entering function: %s", common.GetFunctionName())
err := common.GoldenDiff(func() {
for _, sql := range common.TestSQLs {
fmt.Println(sql)
fmt.Println(SchemaMetaInfo(sql, "sakila"))
}
}, t.Name(), update)
if nil != err {
t.Fatal(err)
}
common.Log.Debug("Exiting function: %s", common.GetFunctionName())
}
......@@ -42,6 +42,7 @@ func main() {
var alterSQLs []string // 待评审的 SQL 中所有 ALTER 请求
alterTableTimes := make(map[string]int) // 待评审的 SQL 中同一经表 ALTER 请求计数器
suggestMerged := make(map[string]map[string]advisor.Rule) // 优化建议去重, key 为 sql 的 fingerprint.ID
tables := make(map[string][]string) // SQL 使用的库表名
// 配置文件&命令行参数解析
initConfig()
......@@ -196,9 +197,15 @@ func main() {
if common.Config.OnlySyntaxCheck {
continue
}
// +++++++++++++++++++++语法检查[结束]+++++++++++++++++++++++}
switch common.Config.ReportType {
case "tables":
env.ChangeDB(vEnv.Connector, q.Query)
tables[id] = ast.SchemaMetaInfo(sql, vEnv.Database)
continue
}
// +++++++++++++++++++++启发式规则建议[开始]+++++++++++++++++++++++{
common.Log.Debug("start of heuristic advisor Query: %s", q.Query)
for item, rule := range advisor.HeuristicRules {
......@@ -377,6 +384,7 @@ func main() {
suggestMerged[id] = sug
switch common.Config.ReportType {
case "json":
case "tables":
case "duplicate-key-checker":
case "rewrite":
case "lint":
......@@ -425,5 +433,16 @@ func main() {
return
}
// 以 JSON 格式输出 SQL 影响的库表名
if common.Config.ReportType == "tables" {
js, err := json.MarshalIndent(tables, "", " ")
if err == nil {
fmt.Println(string(js))
} else {
common.Log.Error("FormatSuggest json.Marshal Error: %v", err)
}
return
}
verboseInfo()
}
......@@ -32,7 +32,7 @@ import (
"time"
"github.com/go-sql-driver/mysql"
"gopkg.in/yaml.v2"
yaml "gopkg.in/yaml.v2"
)
var (
......@@ -669,6 +669,9 @@ func readCmdFlags() error {
}
Config.ReportType = strings.ToLower(*reportType)
if Config.ReportType == "tables" && Config.TestDSN.Schema == "information_schema" {
Config.TestDSN.Schema = "unknown"
}
Config.ReportCSS = *reportCSS
Config.ReportJavascript = *reportJavascript
Config.ReportTitle = *reportTitle
......@@ -842,11 +845,26 @@ var ReportTypes = []ReportType{
Description: "输出 SQL 的抽象语法树,主要用于测试",
Example: `echo "select * from film" | soar -report-type ast`,
},
{
Name: "ast-json",
Description: "以 JSON 格式输出 SQL 的抽象语法树,主要用于测试",
Example: `echo "select * from film" | soar -report-type ast-json`,
},
{
Name: "tiast",
Description: "输出 SQL 的 TiDB抽象语法树,主要用于测试",
Example: `echo "select * from film" | soar -report-type tiast`,
},
{
Name: "tiast-json",
Description: "以 JSON 格式输出 SQL 的 TiDB抽象语法树,主要用于测试",
Example: `echo "select * from film" | soar -report-type tiast-json`,
},
{
Name: "tables",
Description: "以 JSON 格式输出 SQL 使用的库表名",
Example: `echo "select * from film" | soar -report-type meta`,
},
{
Name: "fingerprint",
Description: "输出SQL的指纹",
......
[{
"Collate":{
"Collate":{
"key":"value"
}
}
} {
"Collate":{
"key":"value"
}
} {
"key":"value"
}]
[McLaughlin Hunter Harold]
[{
"title": "Sample Konfabulator Widget",
......
......@@ -34,6 +34,14 @@ echo "select * from film" | soar -rewrite-rules star2columns,delimiter -report-t
```bash
echo "select * from film" | soar -report-type ast
```
## ast-json
* **Description**:以 JSON 格式输出 SQL 的抽象语法树,主要用于测试
* **Example**:
```bash
echo "select * from film" | soar -report-type ast-json
```
## tiast
* **Description**:输出 SQL 的 TiDB抽象语法树,主要用于测试
......@@ -42,6 +50,22 @@ echo "select * from film" | soar -report-type ast
```bash
echo "select * from film" | soar -report-type tiast
```
## tiast-json
* **Description**:以 JSON 格式输出 SQL 的 TiDB抽象语法树,主要用于测试
* **Example**:
```bash
echo "select * from film" | soar -report-type tiast-json
```
## meta
* **Description**:以 JSON 格式输出 SQL 使用的库表名
* **Example**:
```bash
echo "select * from film" | soar -report-type meta
```
## fingerprint
* **Description**:输出SQL的指纹
......
......@@ -113,12 +113,12 @@ func jsonFind(json string, name string, find *[]string) (next []string) {
res.ForEach(func(key, value gjson.Result) bool {
if key.String() == name {
*find = append(*find, value.String())
} else {
switch value.Type {
case gjson.Number, gjson.True, gjson.False, gjson.Null:
default:
next = append(next, value.String())
}
}
switch value.Type {
case gjson.Number, gjson.True, gjson.False, gjson.Null:
default:
// String, JSON
next = append(next, value.String())
}
return true // keep iterating
})
......@@ -138,3 +138,19 @@ func JSONFind(json string, name string) []string {
}
return find
}
// RemoveDuplicatesItem remove duplicate item from list
func RemoveDuplicatesItem(duplicate []string) []string {
m := make(map[string]bool)
for _, item := range duplicate {
if _, ok := m[item]; !ok {
m[item] = true
}
}
var unique []string
for item := range m {
unique = append(unique, item)
}
return unique
}
......@@ -55,6 +55,17 @@ func TestCaptureOutput(t *testing.T) {
func TestJSONFind(t *testing.T) {
Log.Debug("Entering function: %s", GetFunctionName())
jsons := []string{
`
{
"Collate":{
"Collate":{
"Collate":{
"key":"value"
}
}
}
}
`,
`{
"programmers": [
{
......@@ -389,3 +400,12 @@ func TestJSONFind(t *testing.T) {
}
Log.Debug("Exiting function: %s", GetFunctionName())
}
func TestRemoveDuplicatesItem(t *testing.T) {
Log.Debug("Entering function: %s", GetFunctionName())
unique := RemoveDuplicatesItem([]string{"a", "a", "b", "c"})
if len(unique) != 3 {
t.Error("string list length should 3", unique)
}
Log.Debug("Exiting function: %s", GetFunctionName())
}
......@@ -34,6 +34,14 @@ echo "select * from film" | soar -rewrite-rules star2columns,delimiter -report-t
```bash
echo "select * from film" | soar -report-type ast
```
## ast-json
* **Description**:以 JSON 格式输出 SQL 的抽象语法树,主要用于测试
* **Example**:
```bash
echo "select * from film" | soar -report-type ast-json
```
## tiast
* **Description**:输出 SQL 的 TiDB抽象语法树,主要用于测试
......@@ -42,6 +50,22 @@ echo "select * from film" | soar -report-type ast
```bash
echo "select * from film" | soar -report-type tiast
```
## tiast-json
* **Description**:以 JSON 格式输出 SQL 的 TiDB抽象语法树,主要用于测试
* **Example**:
```bash
echo "select * from film" | soar -report-type tiast-json
```
## tables
* **Description**:以 JSON 格式输出 SQL 使用的库表名
* **Example**:
```bash
echo "select * from film" | soar -report-type meta
```
## fingerprint
* **Description**:输出SQL的指纹
......
......@@ -194,6 +194,21 @@ func (vEnv *VirtualEnv) CleanupTestDatabase() {
common.Log.Debug("CleanupTestDatabase done")
}
// ChangeDB use db change dsn Database
func ChangeDB(env *database.Connector, sql string) {
stmt, err := sqlparser.Parse(sql)
if err != nil {
return
}
switch stmt := stmt.(type) {
case *sqlparser.Use:
if stmt.DBName.String() != "" {
env.Database = stmt.DBName.String()
}
}
}
// BuildVirtualEnv rEnv 为 SQL 源环境,DB 使用的信息从接口获取
// 注意:如果是 USE, DDL 等语句,执行完第一条就会返回,后面的 SQL 不会执行
func (vEnv *VirtualEnv) BuildVirtualEnv(rEnv *database.Connector, SQLs ...string) bool {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册