From dda41ec541d251c1dd35318d7e746e685763bb09 Mon Sep 17 00:00:00 2001 From: Leon Zhang Date: Tue, 4 Dec 2018 15:48:57 +0800 Subject: [PATCH] fix #146 ref: https://github.com/XiaoMi/soar/pull/147 Thanks @xiyangxixian --- advisor/testdata/TestListTestSQLs.golden | 1 + ast/testdata/TestCompress.golden | 2 ++ ast/testdata/TestFormat.golden | 8 ++++++++ ast/testdata/TestPretty.golden | 8 ++++++++ ast/testdata/TestTokenize.golden | 10 ++++++---- ast/token.go | 11 +++++++---- common/cases.go | 3 +++ 7 files changed, 35 insertions(+), 8 deletions(-) diff --git a/advisor/testdata/TestListTestSQLs.golden b/advisor/testdata/TestListTestSQLs.golden index cbfbe21..2f4a8bd 100644 --- a/advisor/testdata/TestListTestSQLs.golden +++ b/advisor/testdata/TestListTestSQLs.golden @@ -81,3 +81,4 @@ alter table inventory add index `idx_store_film` (`store_id`,`film_id`); alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`); SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d') create table hello.t (id int unsigned); +select * from tb where data >= '' diff --git a/ast/testdata/TestCompress.golden b/ast/testdata/TestCompress.golden index cc42297..203ba19 100644 --- a/ast/testdata/TestCompress.golden +++ b/ast/testdata/TestCompress.golden @@ -164,3 +164,5 @@ SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d') create table hello.t (id int unsigned); create table hello.t (id int unsigned); +select * from tb where data >= '' +select * from tb where data >= '' diff --git a/ast/testdata/TestFormat.golden b/ast/testdata/TestFormat.golden index f434a85..4236be0 100644 --- a/ast/testdata/TestFormat.golden +++ b/ast/testdata/TestFormat.golden @@ -867,3 +867,11 @@ ORDER BY ) create table hello.t (id int unsigned); create table hello. t (id int unsigned); +select * from tb where data >= '' + +SELECT + * +FROM + tb +WHERE + data >= '' diff --git a/ast/testdata/TestPretty.golden b/ast/testdata/TestPretty.golden index 8af118b..2f6c85a 100644 --- a/ast/testdata/TestPretty.golden +++ b/ast/testdata/TestPretty.golden @@ -1511,3 +1511,11 @@ ORDER BY ) create table hello.t (id int unsigned); create table hello. t (id int unsigned); +select * from tb where data >= '' + +SELECT + * +FROM + tb +WHERE + data >= '' diff --git a/ast/testdata/TestTokenize.golden b/ast/testdata/TestTokenize.golden index 5e3c3b7..b8abe22 100644 --- a/ast/testdata/TestTokenize.golden +++ b/ast/testdata/TestTokenize.golden @@ -5,9 +5,9 @@ SELECT * FROM film WHERE length IS NULL; SELECT * FROM film HAVING title = 'abc'; [{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 film 0} {5 HAVING 0} {1 title 0} {7 = 0} {0 0} {2 'abc' 0} {7 ; 0}] SELECT * FROM sakila.film WHERE length >= 60; -[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 sakila. 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 > 0} {7 = 0} {0 0} {10 60; 0}] +[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 sakila. 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 >= 0} {0 0} {10 60; 0}] SELECT * FROM sakila.film WHERE length >= '60'; -[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 sakila. 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 > 0} {7 = 0} {0 0} {2 '60' 0} {7 ; 0}] +[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 sakila. 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 >= 0} {0 0} {2 '60' 0} {7 ; 0}] SELECT * FROM film WHERE length BETWEEN 60 AND 84; [{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {1 BETWEEN 0} {10 60 0} {6 AND 0} {10 84; 0}] SELECT * FROM film WHERE title LIKE 'AIR%'; @@ -25,7 +25,7 @@ SELECT * FROM film WHERE length > 100 and language_id < 10; SELECT release_year, sum(length) FROM film WHERE length = 123 AND language_id = 1 GROUP BY release_year; [{5 SELECT 0} {1 release_year, 0} {0 0} {4 SUM( 0} {4 LENGTH) 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 = 0} {0 0} {10 123 0} {6 AND 0} {1 language_id 0} {7 = 0} {0 0} {10 1 0} {5 GROUP BY 0} {1 release_year; 0}] SELECT release_year, sum(length) FROM film WHERE length >= 123 GROUP BY release_year; -[{5 SELECT 0} {1 release_year, 0} {0 0} {4 SUM( 0} {4 LENGTH) 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 > 0} {7 = 0} {0 0} {10 123 0} {5 GROUP BY 0} {1 release_year; 0}] +[{5 SELECT 0} {1 release_year, 0} {0 0} {4 SUM( 0} {4 LENGTH) 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {4 LENGTH 0} {7 >= 0} {0 0} {10 123 0} {5 GROUP BY 0} {1 release_year; 0}] SELECT release_year, language_id, sum(length) FROM film GROUP BY release_year, language_id; [{5 SELECT 0} {1 release_year, 0} {0 0} {1 language_id, 0} {0 0} {4 SUM( 0} {4 LENGTH) 0} {0 0} {5 FROM 0} {1 film 0} {5 GROUP BY 0} {1 release_year, 0} {0 0} {1 language_id; 0}] SELECT release_year, sum(length) FROM film WHERE length = 123 GROUP BY release_year,(length+language_id); @@ -57,7 +57,7 @@ SELECT * FROM film WHERE length < 100 ORDER BY length LIMIT 10; SELECT * FROM customer WHERE address_id in (224,510) ORDER BY last_name; [{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 customer 0} {5 WHERE 0} {1 address_id 0} {1 in 0} {7 ( 0} {10 224, 0} {10 510) 0} {0 0} {5 ORDER BY 0} {1 last_name; 0}] SELECT * FROM film WHERE release_year = 2016 AND length != 1 ORDER BY title; -[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {1 release_year 0} {7 = 0} {0 0} {10 2016 0} {6 AND 0} {4 LENGTH 0} {7 ! 0} {7 = 0} {0 0} {10 1 0} {5 ORDER BY 0} {1 title; 0}] +[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {1 release_year 0} {7 = 0} {0 0} {10 2016 0} {6 AND 0} {4 LENGTH 0} {7 != 0} {0 0} {10 1 0} {5 ORDER BY 0} {1 title; 0}] SELECT title FROM film WHERE release_year = 1995; [{5 SELECT 0} {1 title 0} {5 FROM 0} {1 film 0} {5 WHERE 0} {1 release_year 0} {7 = 0} {0 0} {10 1995; 0}] SELECT title, replacement_cost FROM film WHERE language_id = 5 AND length = 70; @@ -164,3 +164,5 @@ SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal [{5 SELECT 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0 0} {2 '%Y-%m-%d' 0} {7 ) 0} {7 , 0} {0 0} {4 COUNT( 0} {1 DISTINCT 0} {7 ( 0} {1 t. 0} {1 usr) 0} {7 ) 0} {0 0} {5 FROM 0} {1 usr_terminal 0} {1 t 0} {5 WHERE 0} {1 t. 0} {1 atm 0} {7 > 0} {0 0} {2 '2018-10-22 00:00:00' 0} {0 0} {6 AND 0} {1 t. 0} {1 agent 0} {1 LIKE 0} {2 '%Chrome%' 0} {0 0} {6 AND 0} {1 t. 0} {1 system 0} {7 = 0} {0 0} {2 'eip' 0} {0 0} {5 GROUP BY 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0 0} {2 '%Y-%m-%d' 0} {7 ) 0} {0 0} {5 ORDER BY 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0 0} {2 '%Y-%m-%d' 0} {7 ) 0}] create table hello.t (id int unsigned); [{1 create 0} {1 table 0} {1 hello. 0} {1 t 0} {7 ( 0} {1 id 0} {1 int 0} {1 unsigned) 0} {7 ; 0}] +select * from tb where data >= '' +[{5 SELECT 0} {7 * 0} {0 0} {5 FROM 0} {1 tb 0} {5 WHERE 0} {1 data 0} {7 >= 0} {0 0} {2 '' 0}] diff --git a/ast/token.go b/ast/token.go index ea64ee9..4d1d2ec 100644 --- a/ast/token.go +++ b/ast/token.go @@ -48,7 +48,12 @@ var cacheHits int var cacheMisses int var tokenCache map[string]Token -var tokenBoundaries = []string{",", ";", ":", ")", "(", ".", "=", "<", ">", "+", "-", "*", "/", "!", "^", "%", "|", "&", "#"} +var tokenBoundaries = []string{ + // multi character + "(>=)", "(<=)", "(!=)", "(<>)", + // single characters + ",", ";", ":", "\\)", "\\(", "\\.", "=", "<", ">", "\\+", "-", "\\*", "/", "!", "\\^", "%", "\\|", "&", "#", +} var tokenReserved = []string{ "ACCESSIBLE", "ACTION", "AGAINST", "AGGREGATE", "ALGORITHM", "ALL", "ALTER", "ANALYSE", "ANALYZE", "AS", "ASC", @@ -124,9 +129,7 @@ var regFunctionString string func init() { var regs []string - for _, reg := range tokenBoundaries { - regs = append(regs, regexp.QuoteMeta(reg)) - } + regs = append(regs, tokenBoundaries...) regBoundariesString = "(" + strings.Join(regs, "|") + ")" regs = make([]string, 0) diff --git a/common/cases.go b/common/cases.go index 25763f7..79cda13 100644 --- a/common/cases.go +++ b/common/cases.go @@ -199,5 +199,8 @@ func init() { `SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')`, // https://github.com/XiaoMi/soar/issues/17 "create table hello.t (id int unsigned);", + + // https://github.com/XiaoMi/soar/issues/146 + "select * from tb where data >= ''", } } -- GitLab