fix #146

  ref: https://github.com/XiaoMi/soar/pull/147

  Thanks @xiyangxixian

advisor/testdata/TestListTestSQLs.golden

@@ -81,3 +81,4 @@ alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
 alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
 SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
 create table hello.t (id int unsigned);
+select * from tb where data >= ''

ast/testdata/TestCompress.golden

@@ -164,3 +164,5 @@ SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal
 SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
 create table hello.t (id int unsigned);
 create table hello.t (id int unsigned);
+select * from tb where data >= ''
+select * from tb where data >= ''

ast/testdata/TestFormat.golden

@@ -867,3 +867,11 @@ ORDER BY
 )
 create table hello.t (id int unsigned);
 create  table  hello. t  (id  int  unsigned);
+select * from tb where data >= ''
+
+SELECT  
+  * 
+FROM  
+  tb  
+WHERE  
+  data  >= ''

ast/testdata/TestPretty.golden

@@ -1511,3 +1511,11 @@ ORDER BY
 )
 create table hello.t (id int unsigned);
 create  table  hello. t  (id  int  unsigned);
+select * from tb where data >= ''
+
+SELECT  
+  * 
+FROM  
+  tb  
+WHERE  
+  data  >= ''

ast/testdata/TestTokenize.golden

@@ -5,9 +5,9 @@ SELECT * FROM film WHERE length IS NULL;
 SELECT * FROM film HAVING title = 'abc';
 [{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 HAVING  0} {1 title  0} {7 = 0} {0   0} {2 'abc' 0} {7 ; 0}]
 SELECT * FROM sakila.film WHERE length >= 60;
-[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 sakila. 0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {7 = 0} {0   0} {10 60; 0}]
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 sakila. 0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 >= 0} {0   0} {10 60; 0}]
 SELECT * FROM sakila.film WHERE length >= '60';
-[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 sakila. 0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {7 = 0} {0   0} {2 '60' 0} {7 ; 0}]
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 sakila. 0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 >= 0} {0   0} {2 '60' 0} {7 ; 0}]
 SELECT * FROM film WHERE length BETWEEN 60 AND 84;
 [{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {1 BETWEEN  0} {10 60  0} {6 AND  0} {10 84; 0}]
 SELECT * FROM film WHERE title LIKE 'AIR%';
@@ -25,7 +25,7 @@ SELECT * FROM film WHERE length > 100 and language_id < 10;
 SELECT release_year, sum(length) FROM film WHERE length = 123 AND language_id = 1 GROUP BY release_year;
 [{5 SELECT  0} {1 release_year, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 123  0} {6 AND  0} {1 language_id  0} {7 = 0} {0   0} {10 1  0} {5 GROUP BY  0} {1 release_year; 0}]
 SELECT release_year, sum(length) FROM film WHERE length >= 123 GROUP BY release_year;
-[{5 SELECT  0} {1 release_year, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {7 = 0} {0   0} {10 123  0} {5 GROUP BY  0} {1 release_year; 0}]
+[{5 SELECT  0} {1 release_year, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 >= 0} {0   0} {10 123  0} {5 GROUP BY  0} {1 release_year; 0}]
 SELECT release_year, language_id, sum(length) FROM film GROUP BY release_year, language_id;
 [{5 SELECT  0} {1 release_year, 0} {0   0} {1 language_id, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 GROUP BY  0} {1 release_year, 0} {0   0} {1 language_id; 0}]
 SELECT release_year, sum(length) FROM film WHERE length = 123 GROUP BY release_year,(length+language_id);
@@ -57,7 +57,7 @@ SELECT * FROM film WHERE length < 100 ORDER BY length LIMIT 10;
 SELECT * FROM customer WHERE address_id in (224,510) ORDER BY last_name;
 [{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 customer  0} {5 WHERE  0} {1 address_id  0} {1 in  0} {7 ( 0} {10 224, 0} {10 510) 0} {0   0} {5 ORDER BY  0} {1 last_name; 0}]
 SELECT * FROM film WHERE release_year = 2016 AND length != 1 ORDER BY title;
-[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 release_year  0} {7 = 0} {0   0} {10 2016  0} {6 AND  0} {4 LENGTH  0} {7 ! 0} {7 = 0} {0   0} {10 1  0} {5 ORDER BY  0} {1 title; 0}]
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 release_year  0} {7 = 0} {0   0} {10 2016  0} {6 AND  0} {4 LENGTH  0} {7 != 0} {0   0} {10 1  0} {5 ORDER BY  0} {1 title; 0}]
 SELECT title FROM film WHERE release_year = 1995;
 [{5 SELECT  0} {1 title  0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 release_year  0} {7 = 0} {0   0} {10 1995; 0}]
 SELECT title, replacement_cost FROM film WHERE language_id = 5 AND length = 70;
@@ -164,3 +164,5 @@ SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal
 [{5 SELECT	 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {7 , 0} {0   0} {4 COUNT( 0} {1 DISTINCT  0} {7 ( 0} {1 t. 0} {1 usr) 0} {7 ) 0} {0   0} {5 FROM  0} {1 usr_terminal  0} {1 t  0} {5 WHERE  0} {1 t. 0} {1 atm  0} {7 > 0} {0   0} {2 '2018-10-22 00:00:00' 0} {0   0} {6 AND  0} {1 t. 0} {1 agent  0} {1 LIKE  0} {2 '%Chrome%' 0} {0   0} {6 AND  0} {1 t. 0} {1 system  0} {7 = 0} {0   0} {2 'eip' 0} {0   0} {5 GROUP BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {0   0} {5 ORDER BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0}]
 create table hello.t (id int unsigned);
 [{1 create  0} {1 table  0} {1 hello. 0} {1 t  0} {7 ( 0} {1 id  0} {1 int  0} {1 unsigned) 0} {7 ; 0}]
+select * from tb where data >= ''
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 tb  0} {5 WHERE  0} {1 data  0} {7 >= 0} {0   0} {2 '' 0}]

ast/token.go

@@ -48,7 +48,12 @@ var cacheHits int
 var cacheMisses int
 var tokenCache map[string]Token
 
-var tokenBoundaries = []string{",", ";", ":", ")", "(", ".", "=", "<", ">", "+", "-", "*", "/", "!", "^", "%", "|", "&", "#"}
+var tokenBoundaries = []string{
+	// multi character
+	"(>=)", "(<=)", "(!=)", "(<>)",
+	// single characters
+	",", ";", ":", "\\)", "\\(", "\\.", "=", "<", ">", "\\+", "-", "\\*", "/", "!", "\\^", "%", "\\|", "&", "#",
+}
 
 var tokenReserved = []string{
 	"ACCESSIBLE", "ACTION", "AGAINST", "AGGREGATE", "ALGORITHM", "ALL", "ALTER", "ANALYSE", "ANALYZE", "AS", "ASC",
@@ -124,9 +129,7 @@ var regFunctionString string
 
 func init() {
 	var regs []string
-	for _, reg := range tokenBoundaries {
-		regs = append(regs, regexp.QuoteMeta(reg))
-	}
+	regs = append(regs, tokenBoundaries...)
 	regBoundariesString = "(" + strings.Join(regs, "|") + ")"
 
 	regs = make([]string, 0)

common/cases.go

@@ -199,5 +199,8 @@ func init() {
 		`SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')`,
 		// https://github.com/XiaoMi/soar/issues/17
 		"create table hello.t (id int unsigned);",
+
+		// https://github.com/XiaoMi/soar/issues/146
+		"select * from tb where data >= ''",
 	}
 }