diff --git a/advisor/testdata/TestListTestSQLs.golden b/advisor/testdata/TestListTestSQLs.golden index cbfbe210750aef574f86fa99a9c1453fdbc7731b..2f4a8bdd6705d4c81ecedfbd7a879b19c32e886b 100644 --- a/advisor/testdata/TestListTestSQLs.golden +++ b/advisor/testdata/TestListTestSQLs.golden @@ -81,3 +81,4 @@ alter table inventory add index `idx_store_film` (`store_id`,`film_id`); alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`); SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d') create table hello.t (id int unsigned); +select * from tb where data >= '' diff --git a/ast/testdata/TestCompress.golden b/ast/testdata/TestCompress.golden index cc42297127d26c7e04919f117804e847283f5ace..203ba195de92542534e7776990bffd9062e59c3f 100644 --- a/ast/testdata/TestCompress.golden +++ b/ast/testdata/TestCompress.golden @@ -164,3 +164,5 @@ SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d') create table hello.t (id int unsigned); create table hello.t (id int unsigned); +select * from tb where data >= '' +select * from tb where data >= '' diff --git a/ast/testdata/TestFormat.golden b/ast/testdata/TestFormat.golden index f434a857aa6291985fda9f237558a2512354236d..4236be095ee94438abf15642d2268a753a8c01ee 100644 --- a/ast/testdata/TestFormat.golden +++ b/ast/testdata/TestFormat.golden @@ -867,3 +867,11 @@ ORDER BY ) create table hello.t (id int unsigned); create table hello. t (id int unsigned); +select * from tb where data >= '' + +SELECT + * +FROM + tb +WHERE + data >= '' diff --git a/ast/testdata/TestPretty.golden b/ast/testdata/TestPretty.golden index 8af118b34be0fb8547d296d9430c87a8b97965c7..2f6c85add72ffc32dbbccc8156cd612fdd165018 100644 --- a/ast/testdata/TestPretty.golden +++ b/ast/testdata/TestPretty.golden @@ -1511,3 +1511,11 @@ ORDER BY ) create table hello.t (id int unsigned); create table hello. t (id int unsigned); +select * from tb where data >= '' + +SELECT + * +FROM + tb +WHERE + data >= '' diff --git a/ast/testdata/TestTokenize.golden b/ast/testdata/TestTokenize.golden index 5e3c3b7bc2be368b3e3330a72ace075d9ab44996..b8abe22ea9a0b16bfe914fe826d2a2fbdc75b014 100644 --- a/ast/testdata/TestTokenize.golden +++ b/ast/testdata/TestTokenize.golden @@ -5,9 +5,9 @@ SELECT * FROM film WHERE length IS NULL; SELECT * FROM film HAVING title = 'abc'; [{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 film 0} {5 HAVING 0} {1 title 0} {7 = 0} {0 0} {2 'abc' 0} {7 ; 0}] SELECT * FROM sakila.film WHERE length >= 60; -[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 sakila. 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 > 0} {7 = 0} {0 0} {10 60; 0}] +[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 sakila. 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 >= 0} {0 0} {10 60; 0}] SELECT * FROM sakila.film WHERE length >= '60'; -[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 sakila. 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 > 0} {7 = 0} {0 0} {2 '60' 0} {7 ; 0}] +[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 sakila. 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 >= 0} {0 0} {2 '60' 0} {7 ; 0}] SELECT * FROM film WHERE length BETWEEN 60 AND 84; [{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {1 BETWEEN 0} {10 60 0} {6 AND 0} {10 84; 0}] SELECT * FROM film WHERE title LIKE 'AIR%'; @@ -25,7 +25,7 @@ SELECT * FROM film WHERE length > 100 and language_id < 10; SELECT release_year, sum(length) FROM film WHERE length = 123 AND language_id = 1 GROUP BY release_year; [{5 SELECT 0} {1 release_year, 0} {0 0} {4 SUM( 0} {4 LENGTH) 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 = 0} {0 0} {10 123 0} {6 AND 0} {1 language_id 0} {7 = 0} {0 0} {10 1 0} {5 GROUP BY 0} {1 release_year; 0}] SELECT release_year, sum(length) FROM film WHERE length >= 123 GROUP BY release_year; -[{5 SELECT 0} {1 release_year, 0} {0 0} {4 SUM( 0} {4 LENGTH) 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 > 0} {7 = 0} {0 0} {10 123 0} {5 GROUP BY 0} {1 release_year; 0}] +[{5 SELECT 0} {1 release_year, 0} {0 0} {4 SUM( 0} {4 LENGTH) 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 >= 0} {0 0} {10 123 0} {5 GROUP BY 0} {1 release_year; 0}] SELECT release_year, language_id, sum(length) FROM film GROUP BY release_year, language_id; [{5 SELECT 0} {1 release_year, 0} {0 0} {1 language_id, 0} {0 0} {4 SUM( 0} {4 LENGTH) 0} {0 0} {5 FROM 0} {1 film 0} {5 GROUP BY 0} {1 release_year, 0} {0 0} {1 language_id; 0}] SELECT release_year, sum(length) FROM film WHERE length = 123 GROUP BY release_year,(length+language_id); @@ -57,7 +57,7 @@ SELECT * FROM film WHERE length < 100 ORDER BY length LIMIT 10; SELECT * FROM customer WHERE address_id in (224,510) ORDER BY last_name; [{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 customer 0} {5 WHERE 0} {1 address_id 0} {1 in 0} {7 ( 0} {10 224, 0} {10 510) 0} {0 0} {5 ORDER BY 0} {1 last_name; 0}] SELECT * FROM film WHERE release_year = 2016 AND length != 1 ORDER BY title; -[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {1 release_year 0} {7 = 0} {0 0} {10 2016 0} {6 AND 0} {4 LENGTH 0} {7 ! 0} {7 = 0} {0 0} {10 1 0} {5 ORDER BY 0} {1 title; 0}] +[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {1 release_year 0} {7 = 0} {0 0} {10 2016 0} {6 AND 0} {4 LENGTH 0} {7 != 0} {0 0} {10 1 0} {5 ORDER BY 0} {1 title; 0}] SELECT title FROM film WHERE release_year = 1995; [{5 SELECT 0} {1 title 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {1 release_year 0} {7 = 0} {0 0} {10 1995; 0}] SELECT title, replacement_cost FROM film WHERE language_id = 5 AND length = 70; @@ -164,3 +164,5 @@ SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal [{5 SELECT 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0 0} {2 '%Y-%m-%d' 0} {7 ) 0} {7 , 0} {0 0} {4 COUNT( 0} {1 DISTINCT 0} {7 ( 0} {1 t. 0} {1 usr) 0} {7 ) 0} {0 0} {5 FROM 0} {1 usr_terminal 0} {1 t 0} {5 WHERE 0} {1 t. 0} {1 atm 0} {7 > 0} {0 0} {2 '2018-10-22 00:00:00' 0} {0 0} {6 AND 0} {1 t. 0} {1 agent 0} {1 LIKE 0} {2 '%Chrome%' 0} {0 0} {6 AND 0} {1 t. 0} {1 system 0} {7 = 0} {0 0} {2 'eip' 0} {0 0} {5 GROUP BY 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0 0} {2 '%Y-%m-%d' 0} {7 ) 0} {0 0} {5 ORDER BY 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0 0} {2 '%Y-%m-%d' 0} {7 ) 0}] create table hello.t (id int unsigned); [{1 create 0} {1 table 0} {1 hello. 0} {1 t 0} {7 ( 0} {1 id 0} {1 int 0} {1 unsigned) 0} {7 ; 0}] +select * from tb where data >= '' +[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 tb 0} {5 WHERE 0} {1 data 0} {7 >= 0} {0 0} {2 '' 0}] diff --git a/ast/token.go b/ast/token.go index ea64ee9a61ac7cd2fd4079e21ed31c4fc4993ce0..4d1d2ecc1ea1fc76a7e87b86e0edc4b797aa5563 100644 --- a/ast/token.go +++ b/ast/token.go @@ -48,7 +48,12 @@ var cacheHits int var cacheMisses int var tokenCache map[string]Token -var tokenBoundaries = []string{",", ";", ":", ")", "(", ".", "=", "<", ">", "+", "-", "*", "/", "!", "^", "%", "|", "&", "#"} +var tokenBoundaries = []string{ + // multi character + "(>=)", "(<=)", "(!=)", "(<>)", + // single characters + ",", ";", ":", "\\)", "\\(", "\\.", "=", "<", ">", "\\+", "-", "\\*", "/", "!", "\\^", "%", "\\|", "&", "#", +} var tokenReserved = []string{ "ACCESSIBLE", "ACTION", "AGAINST", "AGGREGATE", "ALGORITHM", "ALL", "ALTER", "ANALYSE", "ANALYZE", "AS", "ASC", @@ -124,9 +129,7 @@ var regFunctionString string func init() { var regs []string - for _, reg := range tokenBoundaries { - regs = append(regs, regexp.QuoteMeta(reg)) - } + regs = append(regs, tokenBoundaries...) regBoundariesString = "(" + strings.Join(regs, "|") + ")" regs = make([]string, 0) diff --git a/common/cases.go b/common/cases.go index 25763f7d831b9aa29ec9adc107fa0eb3e1fbdfc9..79cda1384a82deee20d7d6ea0f5d957392d731ca 100644 --- a/common/cases.go +++ b/common/cases.go @@ -199,5 +199,8 @@ func init() { `SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')`, // https://github.com/XiaoMi/soar/issues/17 "create table hello.t (id int unsigned);", + + // https://github.com/XiaoMi/soar/issues/146 + "select * from tb where data >= ''", } }