diff --git a/advisor/rules.go b/advisor/rules.go index f1e049402996e26a670032f541c5fe3599bd287c..a617f8f91da0c20b29729f6eb3e7ee79cc24f4ad 100644 --- a/advisor/rules.go +++ b/advisor/rules.go @@ -1098,8 +1098,8 @@ func InBlackList(sql string) bool { } // FormatSuggest 格式化输出优化建议 -// 目前支持：json, text两种形式，其他形式会给结构体的pretty.Println func FormatSuggest(sql string, format string, suggests ...map[string]Rule) (map[string]Rule, string) { + common.Log.Debug("FormatSuggest, Query: %s", sql) var fingerprint, id string var buf []string var score = 100 @@ -1145,7 +1145,7 @@ func FormatSuggest(sql string, format string, suggests ...map[string]Rule) (map[ delete(suggest, k) } } - + common.Log.Debug("FormatSuggest, format: %s", format) switch format { case "json": js, err := json.MarshalIndent(Result{ @@ -1192,6 +1192,7 @@ func FormatSuggest(sql string, format string, suggests ...map[string]Rule) (map[ } } // MySQL + common.Log.Debug("FormatSuggest, start of sortedMySQLSuggest") var sortedMySQLSuggest []string for item := range suggest { if strings.HasPrefix(item, "ERR") { @@ -1213,6 +1214,7 @@ func FormatSuggest(sql string, format string, suggests ...map[string]Rule) (map[ } // Explain + common.Log.Debug("FormatSuggest, start of sortedExplainSuggest") if suggest["EXP.000"].Item != "" { buf = append(buf, fmt.Sprintln("## ", suggest["EXP.000"].Summary)) buf = append(buf, fmt.Sprintln(suggest["EXP.000"].Content)) @@ -1233,6 +1235,7 @@ func FormatSuggest(sql string, format string, suggests ...map[string]Rule) (map[ } // Profiling + common.Log.Debug("FormatSuggest, start of sortedProfilingSuggest") var sortedProfilingSuggest []string for item := range suggest { if strings.HasPrefix(item, "PRO") { @@ -1249,6 +1252,7 @@ func FormatSuggest(sql string, format string, suggests ...map[string]Rule) (map[ } // Trace + common.Log.Debug("FormatSuggest, start of sortedTraceSuggest") var sortedTraceSuggest []string for item := range suggest { if strings.HasPrefix(item, "TRA") { @@ -1265,6 +1269,7 @@ func FormatSuggest(sql string, format string, suggests ...map[string]Rule) (map[ } // Index + common.Log.Debug("FormatSuggest, start of sortedIdxSuggest") var sortedIdxSuggest []string for item := range suggest { if strings.HasPrefix(item, "IDX") { @@ -1293,6 +1298,7 @@ func FormatSuggest(sql string, format string, suggests ...map[string]Rule) (map[ } // Heuristic + common.Log.Debug("FormatSuggest, start of sortedHeuristicSuggest") var sortedHeuristicSuggest []string for item := range suggest { if !strings.HasPrefix(item, "EXP") && @@ -1321,6 +1327,7 @@ func FormatSuggest(sql string, format string, suggests ...map[string]Rule) (map[ } default: + common.Log.Debug("unknown report-type %s", format) buf = append(buf, fmt.Sprintln("Query: ", sql)) for _, rule := range suggest { buf = append(buf, pretty.Sprint(rule)) @@ -1330,12 +1337,12 @@ func FormatSuggest(sql string, format string, suggests ...map[string]Rule) (map[ // 打分 var str string switch common.Config.ReportType { - case "explain-digest", "lint": - str = strings.Join(buf, "\n") - default: + case "markdown", "html": if len(buf) > 1 { str = buf[0] + "\n" + common.Score(score) + "\n\n" + strings.Join(buf[1:], "\n") } + default: + str = strings.Join(buf, "\n") } return suggest, str diff --git a/advisor/testdata/TestListTestSQLs.golden b/advisor/testdata/TestListTestSQLs.golden index 366996e3893ddc65cd0dc0244290ccfe99a8d679..e7cf593418ab759f144829cee49e6628145b3153 100644 --- a/advisor/testdata/TestListTestSQLs.golden +++ b/advisor/testdata/TestListTestSQLs.golden @@ -1,4 +1,3 @@ -create table hello.t (id int unsigned); SELECT * FROM film WHERE length = 86; SELECT * FROM film WHERE length IS NULL; SELECT * FROM film HAVING title = 'abc'; @@ -80,3 +79,5 @@ SELECT description FROM film WHERE description IN('NEWS','asd') GROUP BY descrip alter table address add index idx_city_id(city_id); alter table inventory add index `idx_store_film` (`store_id`,`film_id`); alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`); +SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d') +create table hello.t (id int unsigned); diff --git a/ast/pretty.go b/ast/pretty.go index 8933513adb5d194b82bef443f5fdd144bc671ef0..5a31d5d2056276ba7cf576d56b30c3a45ebad3bf 100644 --- a/ast/pretty.go +++ b/ast/pretty.go @@ -28,6 +28,7 @@ import ( // Pretty 格式化输出SQL func Pretty(sql string, method string) (output string) { + common.Log.Debug("Pretty, Query: %s, method: %s", sql, method) // 超出 Config.MaxPrettySQLLength 长度的SQL会对其指纹进行pretty if len(sql) > common.Config.MaxPrettySQLLength { fingerprint := query.Fingerprint(sql) @@ -216,6 +217,8 @@ func format(query string) string { if j.Value.(string) == "special" { if indentLevel > 0 { indentLevel-- + } else { + break } } else { break diff --git a/ast/testdata/TestFormat.golden b/ast/testdata/TestFormat.golden new file mode 100644 index 0000000000000000000000000000000000000000..36b8fa1e69596f25d609b5dcbfdd3cb15b462026 --- /dev/null +++ b/ast/testdata/TestFormat.golden @@ -0,0 +1,869 @@ +SELECT * FROM film WHERE length = 86; + +SELECT + * +FROM + film +WHERE + LENGTH = 86; +SELECT * FROM film WHERE length IS NULL; + +SELECT + * +FROM + film +WHERE + LENGTH IS NULL; +SELECT * FROM film HAVING title = 'abc'; + +SELECT + * +FROM + film +HAVING + title = 'abc'; +SELECT * FROM sakila.film WHERE length >= 60; + +SELECT + * +FROM + sakila. film +WHERE + LENGTH >= 60; +SELECT * FROM sakila.film WHERE length >= '60'; + +SELECT + * +FROM + sakila. film +WHERE + LENGTH >= '60'; +SELECT * FROM film WHERE length BETWEEN 60 AND 84; + +SELECT + * +FROM + film +WHERE + LENGTH BETWEEN 60 + AND 84; +SELECT * FROM film WHERE title LIKE 'AIR%'; + +SELECT + * +FROM + film +WHERE + title LIKE 'AIR%'; +SELECT * FROM film WHERE title IS NOT NULL; + +SELECT + * +FROM + film +WHERE + title IS NOT NULL; +SELECT * FROM film WHERE length = 114 and title = 'ALABAMA DEVIL'; + +SELECT + * +FROM + film +WHERE + LENGTH = 114 + AND title = 'ALABAMA DEVIL'; +SELECT * FROM film WHERE length > 100 and title = 'ALABAMA DEVIL'; + +SELECT + * +FROM + film +WHERE + LENGTH > 100 + AND title = 'ALABAMA DEVIL'; +SELECT * FROM film WHERE length > 100 and language_id < 10 and title = 'xyz'; + +SELECT + * +FROM + film +WHERE + LENGTH > 100 + AND language_id < 10 + AND title = 'xyz'; +SELECT * FROM film WHERE length > 100 and language_id < 10; + +SELECT + * +FROM + film +WHERE + LENGTH > 100 + AND language_id < 10; +SELECT release_year, sum(length) FROM film WHERE length = 123 AND language_id = 1 GROUP BY release_year; + +SELECT + release_year, SUM( LENGTH) +FROM + film +WHERE + LENGTH = 123 + AND language_id = 1 +GROUP BY + release_year; +SELECT release_year, sum(length) FROM film WHERE length >= 123 GROUP BY release_year; + +SELECT + release_year, SUM( LENGTH) +FROM + film +WHERE + LENGTH >= 123 +GROUP BY + release_year; +SELECT release_year, language_id, sum(length) FROM film GROUP BY release_year, language_id; + +SELECT + release_year, language_id, SUM( LENGTH) +FROM + film +GROUP BY + release_year, language_id; +SELECT release_year, sum(length) FROM film WHERE length = 123 GROUP BY release_year,(length+language_id); + +SELECT + release_year, SUM( LENGTH) +FROM + film +WHERE + LENGTH = 123 +GROUP BY + release_year, (LENGTH+ language_id); +SELECT release_year, sum(film_id) FROM film GROUP BY release_year; + +SELECT + release_year, SUM( film_id) +FROM + film +GROUP BY + release_year; +SELECT * FROM address GROUP BY address,district; + +SELECT + * +FROM + address +GROUP BY + address, district; +SELECT title FROM film WHERE ABS(language_id) = 3 GROUP BY title; + +SELECT + title +FROM + film +WHERE + ABS( language_id) = 3 +GROUP BY + title; +SELECT language_id FROM film WHERE length = 123 GROUP BY release_year ORDER BY language_id; + +SELECT + language_id +FROM + film +WHERE + LENGTH = 123 +GROUP BY + release_year +ORDER BY + language_id; +SELECT release_year FROM film WHERE length = 123 GROUP BY release_year ORDER BY release_year; + +SELECT + release_year +FROM + film +WHERE + LENGTH = 123 +GROUP BY + release_year +ORDER BY + release_year; +SELECT * FROM film WHERE length = 123 ORDER BY release_year ASC, language_id DESC; + +SELECT + * +FROM + film +WHERE + LENGTH = 123 +ORDER BY + release_year ASC, language_id DESC; +SELECT release_year FROM film WHERE length = 123 GROUP BY release_year ORDER BY release_year LIMIT 10; + +SELECT + release_year +FROM + film +WHERE + LENGTH = 123 +GROUP BY + release_year +ORDER BY + release_year +LIMIT + 10; +SELECT * FROM film WHERE length = 123 ORDER BY release_year LIMIT 10; + +SELECT + * +FROM + film +WHERE + LENGTH = 123 +ORDER BY + release_year +LIMIT + 10; +SELECT * FROM film ORDER BY release_year LIMIT 10; + +SELECT + * +FROM + film +ORDER BY + release_year +LIMIT + 10; +SELECT film_id FROM film ORDER BY release_year LIMIT 10; + +SELECT + film_id +FROM + film +ORDER BY + release_year +LIMIT + 10; +SELECT * FROM film WHERE length > 100 ORDER BY length LIMIT 10; + +SELECT + * +FROM + film +WHERE + LENGTH > 100 +ORDER BY + LENGTH +LIMIT + 10; +SELECT * FROM film WHERE length < 100 ORDER BY length LIMIT 10; + +SELECT + * +FROM + film +WHERE + LENGTH < 100 +ORDER BY + LENGTH +LIMIT + 10; +SELECT * FROM customer WHERE address_id in (224,510) ORDER BY last_name; + +SELECT + * +FROM + customer +WHERE + address_id in (224, 510) +ORDER BY + last_name; +SELECT * FROM film WHERE release_year = 2016 AND length != 1 ORDER BY title; + +SELECT + * +FROM + film +WHERE + release_year = 2016 + AND LENGTH != 1 +ORDER BY + title; +SELECT title FROM film WHERE release_year = 1995; + +SELECT + title +FROM + film +WHERE + release_year = 1995; +SELECT title, replacement_cost FROM film WHERE language_id = 5 AND length = 70; + +SELECT + title, replacement_cost +FROM + film +WHERE + language_id = 5 + AND LENGTH = 70; +SELECT title FROM film WHERE language_id > 5 AND length > 70; + +SELECT + title +FROM + film +WHERE + language_id > 5 + AND LENGTH > 70; +SELECT * FROM film WHERE length = 100 and title = 'xyz' ORDER BY release_year; + +SELECT + * +FROM + film +WHERE + LENGTH = 100 + AND title = 'xyz' +ORDER BY + release_year; +SELECT * FROM film WHERE length > 100 and title = 'xyz' ORDER BY release_year; + +SELECT + * +FROM + film +WHERE + LENGTH > 100 + AND title = 'xyz' +ORDER BY + release_year; +SELECT * FROM film WHERE length > 100 ORDER BY release_year; + +SELECT + * +FROM + film +WHERE + LENGTH > 100 +ORDER BY + release_year; +SELECT * FROM city a INNER JOIN country b ON a.country_id=b.country_id; + +SELECT + * +FROM + city a + INNER JOIN country b ON a. country_id= b. country_id; +SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id; + +SELECT + * +FROM + city a + LEFT JOIN country b ON a. country_id= b. country_id; +SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id; + +SELECT + * +FROM + city a + RIGHT JOIN country b ON a. country_id= b. country_id; +SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id WHERE b.last_update IS NULL; + +SELECT + * +FROM + city a + LEFT JOIN country b ON a. country_id= b. country_id +WHERE + b. last_update IS NULL; +SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id WHERE a.last_update IS NULL; + +SELECT + * +FROM + city a + RIGHT JOIN country b ON a. country_id= b. country_id +WHERE + a. last_update IS NULL; +SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id UNION SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id; + +SELECT + * +FROM + city a + LEFT JOIN country b ON a. country_id= b. country_id +UNION +SELECT + * +FROM + city a + RIGHT JOIN country b ON a. country_id= b. country_id; +SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id WHERE a.last_update IS NULL UNION SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id WHERE b.last_update IS NULL; + +SELECT + * +FROM + city a + RIGHT JOIN country b ON a. country_id= b. country_id +WHERE + a. last_update IS NULL +UNION +SELECT + * +FROM + city a + LEFT JOIN country b ON a. country_id= b. country_id +WHERE + b. last_update IS NULL; +SELECT country_id, last_update FROM city NATURAL JOIN country; + +SELECT + country_id, last_update +FROM + city NATURAL + JOIN country; +SELECT country_id, last_update FROM city NATURAL LEFT JOIN country; + +SELECT + country_id, last_update +FROM + city NATURAL + LEFT JOIN country; +SELECT country_id, last_update FROM city NATURAL RIGHT JOIN country; + +SELECT + country_id, last_update +FROM + city NATURAL + RIGHT JOIN country; +SELECT a.country_id, a.last_update FROM city a STRAIGHT_JOIN country b ON a.country_id=b.country_id; + +SELECT + a. country_id, a. last_update +FROM + city a STRAIGHT_JOIN country b ON a. country_id= b. country_id; +SELECT d.deptno,d.dname,d.loc FROM scott.dept d WHERE d.deptno IN (SELECT e.deptno FROM scott.emp e); + +SELECT + d. deptno, d. dname, d. loc +FROM + scott. dept d +WHERE + d. deptno IN ( +SELECT + e. deptno +FROM + scott. emp e); +SELECT visitor_id, url FROM (SELECT id FROM log WHERE ip="123.45.67.89" order by tsdesc limit 50, 10) I JOIN log ON (I.id=log.id) JOIN url ON (url.id=log.url_id) order by TS desc; + +SELECT + visitor_id, url +FROM + ( +SELECT + id +FROM + LOG +WHERE + ip= "123.45.67.89" +ORDER BY + tsdesc +LIMIT + 50, 10) I + JOIN LOG ON (I. id= LOG. id) + JOIN url ON (url. id= LOG. url_id) +ORDER BY + TS desc; +DELETE city, country FROM city INNER JOIN country using (country_id) WHERE city.city_id = 1; +DELETE city, country +FROM + city + INNER JOIN country using (country_id) +WHERE + city. city_id = 1; +DELETE city FROM city LEFT JOIN country ON city.country_id = country.country_id WHERE country.country IS NULL; +DELETE city +FROM + city + LEFT JOIN country ON city. country_id = country. country_id +WHERE + country. country IS NULL; +DELETE a1, a2 FROM city AS a1 INNER JOIN country AS a2 WHERE a1.country_id=a2.country_id; +DELETE a1, a2 +FROM + city AS a1 + INNER JOIN country AS a2 +WHERE + a1. country_id= a2. country_id; +DELETE FROM a1, a2 USING city AS a1 INNER JOIN country AS a2 WHERE a1.country_id=a2.country_id; + +DELETE FROM + a1, a2 USING city AS a1 + INNER JOIN country AS a2 +WHERE + a1. country_id= a2. country_id; +DELETE FROM film WHERE length > 100; + +DELETE FROM + film +WHERE + LENGTH > 100; +UPDATE city INNER JOIN country USING(country_id) SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.city_id=10; + +UPDATE + city + INNER JOIN country USING( country_id) +SET + city. city = 'Abha', + city. last_update = '2006-02-15 04:45:25', + country. country = 'Afghanistan' +WHERE + city. city_id= 10; +UPDATE city INNER JOIN country ON city.country_id = country.country_id INNER JOIN address ON city.city_id = address.city_id SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.city_id=10; + +UPDATE + city + INNER JOIN country ON city. country_id = country. country_id + INNER JOIN address ON city. city_id = address. city_id +SET + city. city = 'Abha', + city. last_update = '2006-02-15 04:45:25', + country. country = 'Afghanistan' +WHERE + city. city_id= 10; +UPDATE city, country SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.country_id = country.country_id AND city.city_id=10; + +UPDATE + city, country +SET + city. city = 'Abha', + city. last_update = '2006-02-15 04:45:25', + country. country = 'Afghanistan' +WHERE + city. country_id = country. country_id + AND city. city_id= 10; +UPDATE film SET length = 10 WHERE language_id = 20; + +UPDATE + film +SET + LENGTH = 10 +WHERE + language_id = 20; +INSERT INTO city (country_id) SELECT country_id FROM country; +INSERT INTO city (country_id) +SELECT + country_id +FROM + country; +INSERT INTO city (country_id) VALUES (1),(2),(3); +INSERT INTO city (country_id) +VALUES + (1), + (2), + (3); +INSERT INTO city (country_id) VALUES (10); +INSERT INTO city (country_id) +VALUES + (10); +INSERT INTO city (country_id) SELECT 10 FROM DUAL; +INSERT INTO city (country_id) +SELECT + 10 +FROM + DUAL; +REPLACE INTO city (country_id) SELECT country_id FROM country; +REPLACE INTO city (country_id) +SELECT + country_id +FROM + country; +REPLACE INTO city (country_id) VALUES (1),(2),(3); +REPLACE INTO city (country_id) +VALUES + (1), + (2), + (3); +REPLACE INTO city (country_id) VALUES (10); +REPLACE INTO city (country_id) +VALUES + (10); +REPLACE INTO city (country_id) SELECT 10 FROM DUAL; +REPLACE INTO city (country_id) +SELECT + 10 +FROM + DUAL; +SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film; + +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + ( +SELECT + film_id +FROM + film +) film +) film +) film +) film +) film +) film +) film +) film +) film +) film +) film +) film +) film +) film +) film +) film; +SELECT * FROM film WHERE language_id = (SELECT language_id FROM language LIMIT 1); + +SELECT + * +FROM + film +WHERE + language_id = ( +SELECT + language_id +FROM + language +LIMIT + 1); +SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id; + +SELECT + * +FROM + city i + LEFT JOIN country o ON i. city_id= o. country_id +UNION +SELECT + * +FROM + city i + RIGHT JOIN country o ON i. city_id= o. country_id; +SELECT * FROM (SELECT * FROM actor WHERE last_update='2006-02-15 04:34:33' and last_name='CHASE') t WHERE last_update='2006-02-15 04:34:33' and last_name='CHASE' GROUP BY first_name; + +SELECT + * +FROM + ( +SELECT + * +FROM + actor +WHERE + last_update= '2006-02-15 04:34:33' + AND last_name= 'CHASE' +) t +WHERE + last_update= '2006-02-15 04:34:33' + AND last_name= 'CHASE' +GROUP BY + first_name; +SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id; + +SELECT + * +FROM + city i + LEFT JOIN country o ON i. city_id= o. country_id +UNION +SELECT + * +FROM + city i + RIGHT JOIN country o ON i. city_id= o. country_id; +SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id WHERE o.country_id is null union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id WHERE i.city_id is null; + +SELECT + * +FROM + city i + LEFT JOIN country o ON i. city_id= o. country_id +WHERE + o. country_id is null +UNION +SELECT + * +FROM + city i + RIGHT JOIN country o ON i. city_id= o. country_id +WHERE + i. city_id is null; +SELECT first_name,last_name,email FROM customer STRAIGHT_JOIN address ON customer.address_id=address.address_id; + +SELECT + first_name, last_name, email +FROM + customer STRAIGHT_JOIN address ON customer. address_id= address. address_id; +SELECT ID,name FROM (SELECT address FROM customer_list WHERE SID=1 order by phone limit 50,10) a JOIN customer_list l ON (a.address=l.address) JOIN city c ON (c.city=l.city) order by phone desc; + +SELECT + ID, name +FROM + ( +SELECT + address +FROM + customer_list +WHERE + SID= 1 +ORDER BY + phone +LIMIT + 50, 10) a + JOIN customer_list l ON (a. address= l. address) + JOIN city c ON (c. city= l. city) +ORDER BY + phone desc; +SELECT * FROM film WHERE date(last_update)='2006-02-15'; + +SELECT + * +FROM + film +WHERE + DATE( last_update) = '2006-02-15'; +SELECT last_update FROM film GROUP BY date(last_update); + +SELECT + last_update +FROM + film +GROUP BY + DATE( last_update); +SELECT last_update FROM film order by date(last_update); + +SELECT + last_update +FROM + film +ORDER BY + DATE( last_update); +SELECT description FROM film WHERE description IN('NEWS','asd') GROUP BY description; + +SELECT + description +FROM + film +WHERE + description IN( 'NEWS', + 'asd' +) +GROUP BY + description; +alter table address add index idx_city_id(city_id); + +ALTER TABLE + address +ADD + index idx_city_id( city_id); +alter table inventory add index `idx_store_film` (`store_id`,`film_id`); + +ALTER TABLE + inventory +ADD + index `idx_store_film` ( + `store_id`, `film_id`); +alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`); + +ALTER TABLE + inventory +ADD + index `idx_store_film` ( + `store_id`, `film_id`), + ADD + index `idx_store_film` ( + `store_id`, `film_id`), + ADD + index `idx_store_film` ( + `store_id`, `film_id`); +SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d') + +SELECT + DATE_FORMAT( t. atm, '%Y-%m-%d' +), +COUNT( DISTINCT ( + t. usr)) + FROM + usr_terminal t + WHERE + t. atm > '2018-10-22 00:00:00' + AND t. agent LIKE '%Chrome%' + AND t. system = 'eip' + GROUP BY + DATE_FORMAT( t. atm, '%Y-%m-%d' +) +ORDER BY + DATE_FORMAT( t. atm, '%Y-%m-%d' +) +create table hello.t (id int unsigned); +create table hello. t (id int unsigned); diff --git a/ast/testdata/TestPretty.golden b/ast/testdata/TestPretty.golden index a642ec11808a778d20c128bb86f6bab31c105d1c..5eaf8c5b0a0581cf9fd709c1ed615d6df3907acb 100644 --- a/ast/testdata/TestPretty.golden +++ b/ast/testdata/TestPretty.golden @@ -630,8 +630,6 @@ ALTER TABLE t1 CHANGE b a INT NOT NULL; ALTER TABLE t1 CHANGE b a INT NOT NULL; -create table hello.t (id int unsigned); -create table hello. t (id int unsigned); SELECT * FROM film WHERE length = 86; SELECT @@ -1480,3 +1478,24 @@ ADD ADD index `idx_store_film` ( `store_id`, `film_id`); +SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d') + +SELECT + DATE_FORMAT( t. atm, '%Y-%m-%d' +), +COUNT( DISTINCT ( + t. usr)) + FROM + usr_terminal t + WHERE + t. atm > '2018-10-22 00:00:00' + AND t. agent LIKE '%Chrome%' + AND t. system = 'eip' + GROUP BY + DATE_FORMAT( t. atm, '%Y-%m-%d' +) +ORDER BY + DATE_FORMAT( t. atm, '%Y-%m-%d' +) +create table hello.t (id int unsigned); +create table hello. t (id int unsigned); diff --git a/ast/token_test.go b/ast/token_test.go index 08ba3ef8c5d90930457226e77038ac2fe52d698c..f9174eacdcd5f2074d69e436851054e7fccb6934 100644 --- a/ast/token_test.go +++ b/ast/token_test.go @@ -77,9 +77,14 @@ func TestCompress(t *testing.T) { } func TestFormat(t *testing.T) { - for _, sql := range common.TestSQLs { - fmt.Println(sql) - fmt.Println(format(sql)) + err := common.GoldenDiff(func() { + for _, sql := range common.TestSQLs { + fmt.Println(sql) + fmt.Println(format(sql)) + } + }, t.Name(), update) + if nil != err { + t.Fatal(err) } } diff --git a/cmd/soar/soar.go b/cmd/soar/soar.go index b01ba234e22873fb3bb3d00a41095a888c2e8fd0..82a17d153f96ec186c749f0a06f05cd4ac400b20 100644 --- a/cmd/soar/soar.go +++ b/cmd/soar/soar.go @@ -257,6 +257,7 @@ func main() { // +++++++++++++++++++++语法检查[结束]+++++++++++++++++++++++} // +++++++++++++++++++++启发式规则建议[开始]+++++++++++++++++++++++{ + common.Log.Debug("start of heuristic advisor Query: %s", q.Query) for item, rule := range advisor.HeuristicRules { // 去除忽略的建议检查 okFunc := (*advisor.Query4Audit).RuleOK @@ -267,11 +268,13 @@ func main() { } } } + common.Log.Debug("end of heuristic advisor Query: %s", q.Query) // +++++++++++++++++++++启发式规则建议[结束]+++++++++++++++++++++++} // +++++++++++++++++++++索引优化建议[开始]+++++++++++++++++++++++{ // 如果配置了索引建议过滤规则，不进行索引优化建议 // 在配置文件ignore-rules中添加 'IDX.*'即可屏蔽索引优化建议 + common.Log.Debug("start of index advisor Query: %s", q.Query) if !advisor.IsIgnoreRule("IDX.") { if vEnv.BuildVirtualEnv(rEnv, q.Query) { idxAdvisor, err := advisor.NewAdvisor(vEnv, *rEnv, *q) @@ -314,10 +317,12 @@ func main() { common.Log.Error("vEnv.BuildVirtualEnv Error: prepare SQL '%s' in vEnv failed.", q.Query) } } + common.Log.Debug("end of index advisor Query: %s", q.Query) // +++++++++++++++++++++索引优化建议[结束]+++++++++++++++++++++++} // +++++++++++++++++++++EXPLAIN建议[开始]+++++++++++++++++++++++{ // 如果未配置Online或Test无法给Explain建议 + common.Log.Debug("start of explain Query: %s", q.Query) if !common.Config.OnlineDSN.Disable && !common.Config.TestDSN.Disable { // 因为EXPLAIN依赖数据库环境，所以把这段逻辑放在启发式建议和索引建议后面 if common.Config.Explain { @@ -335,6 +340,7 @@ func main() { // EXPLAIN阶段给出的ERROR是ERR.002 mysqlSuggest["ERR.002"] = advisor.RuleMySQLError("ERR.002", err) common.Log.Error("vEnv.Explain Error: %v", err) + continue } } // 分析EXPLAIN结果 @@ -345,9 +351,11 @@ func main() { } } } + common.Log.Debug("end of explain Query: %s", q.Query) // +++++++++++++++++++++EXPLAIN建议[结束]+++++++++++++++++++++++} // +++++++++++++++++++++Profiling[开始]+++++++++++++++++++++++++{ + common.Log.Debug("start of profiling Query: %s", q.Query) if common.Config.Profiling { res, err := vEnv.Profiling(q.Query) if err == nil { @@ -360,9 +368,11 @@ func main() { common.Log.Error("Profiling Error: %v", err) } } + common.Log.Debug("end of profiling Query: %s", q.Query) // +++++++++++++++++++++Profiling[结束]++++++++++++++++++++++++++} // +++++++++++++++++++++Trace [开始]+++++++++++++++++++++++++{ + common.Log.Debug("start of trace Query: %s", q.Query) if common.Config.Trace { res, err := vEnv.Trace(q.Query) if err == nil { @@ -375,9 +385,11 @@ func main() { common.Log.Error("Trace Error: %v", err) } } + common.Log.Debug("end of trace Query: %s", q.Query) // +++++++++++++++++++++Trace [结束]++++++++++++++++++++++++++} // +++++++++++++++++++++SQL重写[开始]+++++++++++++++++++++++++{ + common.Log.Debug("start of rewrite Query: %s", q.Query) if common.Config.ReportType == "rewrite" { if strings.HasPrefix(strings.TrimSpace(strings.ToLower(sql)), "create") || strings.HasPrefix(strings.TrimSpace(strings.ToLower(sql)), "alter") || @@ -411,9 +423,11 @@ func main() { fmt.Println(strings.TrimSpace(rw.NewSQL)) } } + common.Log.Debug("end of rewrite Query: %s", q.Query) // +++++++++++++++++++++SQL重写[结束]++++++++++++++++++++++++++} - // 打印单条SQL优化建议 + // +++++++++++++++++++++打印单条SQL优化建议[开始]++++++++++++++++++++++++++{ + common.Log.Debug("start of print suggestions, Query: %s", q.Query) sug, str := advisor.FormatSuggest(q.Query, common.Config.ReportType, heuristicSuggest, idxSuggest, expSuggest, proSuggest, traceSuggest, mysqlSuggest) suggestMerged[id] = sug switch common.Config.ReportType { @@ -443,6 +457,8 @@ func main() { default: fmt.Println(str) } + common.Log.Debug("end of print suggestions, Query: %s", q.Query) + // +++++++++++++++++++++打印单条SQL优化建议[结束]++++++++++++++++++++++++++} } // 同一张表的多条ALTER语句合并为一条 diff --git a/common/cases.go b/common/cases.go index 48f7dcdc46c9f3447d3e23807208722b52a6343c..4d20ac572ccb31cd1c167638df2a83ffa3e157a0 100644 --- a/common/cases.go +++ b/common/cases.go @@ -23,9 +23,6 @@ func init() { // 所有的SQL都要以分号结尾，-list-test-sqls参数会打印这个list，以分号结尾可方便测试 // 如：./soar -list-test-sql | ./soar TestSQLs = []string{ - // DDL - "create table hello.t (id int unsigned);", - // single equality "SELECT * FROM film WHERE length = 86;", // index(length) "SELECT * FROM film WHERE length IS NULL;", // index(length) @@ -197,5 +194,10 @@ func init() { "alter table address add index idx_city_id(city_id);", "alter table inventory add index `idx_store_film` (`store_id`,`film_id`);", "alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);", + + // https://github.com/XiaoMi/soar/issues/47 + `SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')`, + // https://github.com/XiaoMi/soar/issues/17 + "create table hello.t (id int unsigned);", } } diff --git a/database/explain.go b/database/explain.go index ecaed35f30a5b6c2f58f552b4aa6e97ab0517ef1..635ad7ba5a20f9c29369b3f55d2cb9d4c8d6f52f 100644 --- a/database/explain.go +++ b/database/explain.go @@ -789,6 +789,8 @@ func parseTraditionalExplainText(content string) (explainRows []*ExplainRow, err if err != nil { filtered = 0.00 } + // filtered may larger than 100.00 + // https://bugs.mysql.com/bug.php?id=34124 if filtered > 100.00 { filtered = 100.00 } @@ -981,7 +983,7 @@ func ParseExplainResult(res *QueryResult, formatType int) (exp *ExplainInfo, err // Explain 获取SQL的explain信息 func (db *Connector) Explain(sql string, explainType int, formatType int) (exp *ExplainInfo, err error) { - exp = &ExplainInfo{} + exp = &ExplainInfo{SQL: sql} if explainType != TraditionalExplainType { formatType = TraditionalFormatExplain } @@ -1004,8 +1006,6 @@ func (db *Connector) Explain(sql string, explainType int, formatType int) (exp * // 解析mysql结果，输出ExplainInfo exp, err = ParseExplainResult(res, formatType) - // 补全SQL - exp.SQL = sql return exp, err } diff --git a/doc/structure.md b/doc/structure.md index 60cc5260573054636a0bdae4674b930578300668..00086012b1fe1b53d781dba147bce3aab7cea95d 100644 --- a/doc/structure.md +++ b/doc/structure.md @@ -7,7 +7,7 @@ SOAR主要由语法解析器，集成环境，优化建议，重写逻辑，工 ## 语法解析和语法检查 -一条SQL从文件，标准输入或命令行参数等形式传递给SOAR后首先进入语法解析器，这里一开始我们选用了vitess的语法解析库作为SOAR的语法解析库，但随时需求的不断增加我们发现有些复杂需求使用vitess的语法解析实现起来比较逻辑比较复杂。于是参考业办其他数据库产品，我们引入了TiDB的语法解析器做为补充。我们发现这两个解析库还存在一定的盲区，于是又引入了MySQL执行返回结果作为多版本SQL方言的补充。大家也可以看到在语法解析器这里，SOAR的实现方案是松散的、可插拔的。SOAR并不直接维护庞大的语法解析库，它把各种优秀的语法解析库集成在一起，各取所长。 +一条SQL从文件，标准输入或命令行参数等形式传递给SOAR后首先进入语法解析器，这里一开始我们选用了vitess的语法解析库作为SOAR的语法解析库，但随时需求的不断增加我们发现有些复杂需求使用vitess的语法解析实现起来比较逻辑比较复杂。于是参考业务其他数据库产品，我们引入了TiDB的语法解析器做为补充。我们发现这两个解析库还存在一定的盲区，于是又引入了MySQL执行返回结果作为多版本SQL方言的补充。大家也可以看到在语法解析器这里，SOAR的实现方案是松散的、可插拔的。SOAR并不直接维护庞大的语法解析库，它把各种优秀的语法解析库集成在一起，各取所长。 ## 集成环境