From eeeab8edb120382879fa10750bc3fb079b5833f5 Mon Sep 17 00:00:00 2001
From: Leon Zhang <martianzhang@hotmail.com>
Date: Wed, 21 Nov 2018 22:12:26 +0800
Subject: [PATCH] daily update

  1. go version move into Makefile
  2. add golden test for ast
  3. explainAbleSQL use tidb parser also
---
 Makefile                                |  22 +-
 ast/testdata/TestCompress.golden        | 166 ++++++++++++++
 ast/testdata/TestGetQuotedString.golden |  24 ++
 ast/testdata/TestLeftNewLines.golden    |   3 +
 ast/testdata/TestNewLines.golden        |   3 +
 ast/testdata/TestSplitStatement.golden  |  27 +++
 ast/testdata/TestTokenize.golden        | 166 ++++++++++++++
 ast/testdata/TestTokenizer.golden       | 281 ++++++++++++++++++++++++
 ast/token_test.go                       |  95 +++++---
 database/explain.go                     |  27 ++-
 database/explain_test.go                |   9 +
 genver.sh                               |  15 --
 12 files changed, 790 insertions(+), 48 deletions(-)
 create mode 100644 ast/testdata/TestCompress.golden
 create mode 100644 ast/testdata/TestGetQuotedString.golden
 create mode 100644 ast/testdata/TestLeftNewLines.golden
 create mode 100644 ast/testdata/TestNewLines.golden
 create mode 100644 ast/testdata/TestSplitStatement.golden
 create mode 100644 ast/testdata/TestTokenize.golden
 create mode 100644 ast/testdata/TestTokenizer.golden

diff --git a/Makefile b/Makefile
index f7f7c09..a3db656 100644
--- a/Makefile
+++ b/Makefile
@@ -9,15 +9,13 @@ GOPATH ?= $(shell go env GOPATH)
 ifeq "$(GOPATH)" ""
   $(error Please set the environment variable GOPATH before running `make`)
 endif
+PATH := ${GOPATH}/bin:$(PATH)
 GCFLAGS=-gcflags "all=-trimpath=${GOPATH}"
 LDFLAGS=-ldflags="-s -w"
 
-PATH := ${GOPATH}/bin:$(PATH)
-
 # These are the values we want to pass for VERSION  and BUILD
 BUILD_TIME=`date +%Y%m%d%H%M`
 COMMIT_VERSION=`git rev-parse HEAD`
-GO_VERSION_MIN=1.10
 
 # Add mysql version for testing `MYSQL_RELEASE=percona MYSQL_VERSION=5.7 make docker`
 # MYSQL_RELEASE: mysql, percona, mariadb ...
@@ -29,6 +27,18 @@ MYSQL_VERSION := $(or ${MYSQL_VERSION}, ${MYSQL_VERSION}, latest)
 .PHONY: all
 all: | fmt build
 
+.PHONY: go_version_check
+GO_VERSION_MIN=1.10
+# Parse out the x.y or x.y.z version and output a single value x*10000+y*100+z (e.g., 1.9 is 10900)
+# that allows the three components to be checked in a single comparison.
+VER_TO_INT:=awk '{split(substr($$0, match ($$0, /[0-9\.]+/)), a, "."); print a[1]*10000+a[2]*100+a[3]}'
+go_version_check:
+	@echo "\033[92mGo version check\033[0m"
+	@if test $(shell go version | $(VER_TO_INT) ) -lt \
+  	$(shell echo "$(GO_VERSION_MIN)" | $(VER_TO_INT)); \
+  	then printf "go version $(GO_VERSION_MIN)+ required, found: "; go version; exit 1; \
+		else echo "go version check pass";	fi
+
 # Dependency check
 .PHONY: deps
 deps:
@@ -40,7 +50,7 @@ deps:
 
 # Code format
 .PHONY: fmt
-fmt:
+fmt: go_version_check
 	@echo "\033[92mRun gofmt on all source files ...\033[0m"
 	@echo "gofmt -l -s -w ..."
 	@ret=0 && for d in $$(go list -f '{{.Dir}}' ./... | grep -v /vendor/); do \
@@ -74,7 +84,7 @@ cover: test
 build: fmt tidb-parser
 	@echo "\033[92mBuilding ...\033[0m"
 	@mkdir -p bin
-	@bash ./genver.sh $(GO_VERSION_MIN)
+	@bash ./genver.sh
 	@ret=0 && for d in $$(go list -f '{{if (eq .Name "main")}}{{.ImportPath}}{{end}}' ./...); do \
 		b=$$(basename $${d}) ; \
 		go build ${GCFLAGS} -o bin/$${b} $$d || ret=$$? ; \
@@ -84,7 +94,7 @@ build: fmt tidb-parser
 .PHONY: fast
 fast: fmt
 	@echo "\033[92mBuilding ...\033[0m"
-	@bash ./genver.sh $(GO_VERSION_MIN)
+	@bash ./genver.sh
 	@ret=0 && for d in $$(go list -f '{{if (eq .Name "main")}}{{.ImportPath}}{{end}}' ./...); do \
 		b=$$(basename $${d}) ; \
 		go build ${GCFLAGS} -o bin/$${b} $$d || ret=$$? ; \
diff --git a/ast/testdata/TestCompress.golden b/ast/testdata/TestCompress.golden
new file mode 100644
index 0000000..cc42297
--- /dev/null
+++ b/ast/testdata/TestCompress.golden
@@ -0,0 +1,166 @@
+SELECT * FROM film WHERE length = 86;
+SELECT * FROM film WHERE length = 86;
+SELECT * FROM film WHERE length IS NULL;
+SELECT * FROM film WHERE length IS NULL;
+SELECT * FROM film HAVING title = 'abc';
+SELECT * FROM film HAVING title = 'abc';
+SELECT * FROM sakila.film WHERE length >= 60;
+SELECT * FROM sakila.film WHERE length >= 60;
+SELECT * FROM sakila.film WHERE length >= '60';
+SELECT * FROM sakila.film WHERE length >= '60';
+SELECT * FROM film WHERE length BETWEEN 60 AND 84;
+SELECT * FROM film WHERE length BETWEEN 60 AND 84;
+SELECT * FROM film WHERE title LIKE 'AIR%';
+SELECT * FROM film WHERE title LIKE 'AIR%';
+SELECT * FROM film WHERE title IS NOT NULL;
+SELECT * FROM film WHERE title IS NOT NULL;
+SELECT * FROM film WHERE length = 114 and title = 'ALABAMA DEVIL';
+SELECT * FROM film WHERE length = 114 and title = 'ALABAMA DEVIL';
+SELECT * FROM film WHERE length > 100 and title = 'ALABAMA DEVIL';
+SELECT * FROM film WHERE length > 100 and title = 'ALABAMA DEVIL';
+SELECT * FROM film WHERE length > 100 and language_id < 10 and title = 'xyz';
+SELECT * FROM film WHERE length > 100 and language_id < 10 and title = 'xyz';
+SELECT * FROM film WHERE length > 100 and language_id < 10;
+SELECT * FROM film WHERE length > 100 and language_id < 10;
+SELECT release_year, sum(length) FROM film WHERE length = 123 AND language_id = 1 GROUP BY release_year;
+SELECT release_year, sum(length) FROM film WHERE length = 123 AND language_id = 1 GROUP BY release_year;
+SELECT release_year, sum(length) FROM film WHERE length >= 123 GROUP BY release_year;
+SELECT release_year, sum(length) FROM film WHERE length >= 123 GROUP BY release_year;
+SELECT release_year, language_id, sum(length) FROM film GROUP BY release_year, language_id;
+SELECT release_year, language_id, sum(length) FROM film GROUP BY release_year, language_id;
+SELECT release_year, sum(length) FROM film WHERE length = 123 GROUP BY release_year,(length+language_id);
+SELECT release_year, sum(length) FROM film WHERE length = 123 GROUP BY release_year,(length+language_id);
+SELECT release_year, sum(film_id) FROM film GROUP BY release_year;
+SELECT release_year, sum(film_id) FROM film GROUP BY release_year;
+SELECT * FROM address GROUP BY address,district;
+SELECT * FROM address GROUP BY address,district;
+SELECT title FROM film WHERE ABS(language_id) = 3 GROUP BY title;
+SELECT title FROM film WHERE ABS(language_id) = 3 GROUP BY title;
+SELECT language_id FROM film WHERE length = 123 GROUP BY release_year ORDER BY language_id;
+SELECT language_id FROM film WHERE length = 123 GROUP BY release_year ORDER BY language_id;
+SELECT release_year FROM film WHERE length = 123 GROUP BY release_year ORDER BY release_year;
+SELECT release_year FROM film WHERE length = 123 GROUP BY release_year ORDER BY release_year;
+SELECT * FROM film WHERE length = 123 ORDER BY release_year ASC, language_id DESC;
+SELECT * FROM film WHERE length = 123 ORDER BY release_year ASC, language_id DESC;
+SELECT release_year FROM film WHERE length = 123 GROUP BY release_year ORDER BY release_year LIMIT 10;
+SELECT release_year FROM film WHERE length = 123 GROUP BY release_year ORDER BY release_year LIMIT 10;
+SELECT * FROM film WHERE length = 123 ORDER BY release_year LIMIT 10;
+SELECT * FROM film WHERE length = 123 ORDER BY release_year LIMIT 10;
+SELECT * FROM film ORDER BY release_year LIMIT 10;
+SELECT * FROM film ORDER BY release_year LIMIT 10;
+SELECT film_id FROM film ORDER BY release_year LIMIT 10;
+SELECT film_id FROM film ORDER BY release_year LIMIT 10;
+SELECT * FROM film WHERE length > 100 ORDER BY length LIMIT 10;
+SELECT * FROM film WHERE length > 100 ORDER BY length LIMIT 10;
+SELECT * FROM film WHERE length < 100 ORDER BY length LIMIT 10;
+SELECT * FROM film WHERE length < 100 ORDER BY length LIMIT 10;
+SELECT * FROM customer WHERE address_id in (224,510) ORDER BY last_name;
+SELECT * FROM customer WHERE address_id in (224,510) ORDER BY last_name;
+SELECT * FROM film WHERE release_year = 2016 AND length != 1 ORDER BY title;
+SELECT * FROM film WHERE release_year = 2016 AND length != 1 ORDER BY title;
+SELECT title FROM film WHERE release_year = 1995;
+SELECT title FROM film WHERE release_year = 1995;
+SELECT title, replacement_cost FROM film WHERE language_id = 5 AND length = 70;
+SELECT title, replacement_cost FROM film WHERE language_id = 5 AND length = 70;
+SELECT title FROM film WHERE language_id > 5 AND length > 70;
+SELECT title FROM film WHERE language_id > 5 AND length > 70;
+SELECT * FROM film WHERE length = 100 and title = 'xyz' ORDER BY release_year;
+SELECT * FROM film WHERE length = 100 and title = 'xyz' ORDER BY release_year;
+SELECT * FROM film WHERE length > 100 and title = 'xyz' ORDER BY release_year;
+SELECT * FROM film WHERE length > 100 and title = 'xyz' ORDER BY release_year;
+SELECT * FROM film WHERE length > 100 ORDER BY release_year;
+SELECT * FROM film WHERE length > 100 ORDER BY release_year;
+SELECT * FROM city a INNER JOIN country b ON a.country_id=b.country_id;
+SELECT * FROM city a INNER JOIN country b ON a.country_id=b.country_id;
+SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id;
+SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id;
+SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id;
+SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id;
+SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id WHERE b.last_update IS NULL;
+SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id WHERE b.last_update IS NULL;
+SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id WHERE a.last_update IS NULL;
+SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id WHERE a.last_update IS NULL;
+SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id UNION SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id;
+SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id UNION SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id;
+SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id WHERE a.last_update IS NULL UNION SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id WHERE b.last_update IS NULL;
+SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id WHERE a.last_update IS NULL UNION SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id WHERE b.last_update IS NULL;
+SELECT country_id, last_update FROM city NATURAL JOIN country;
+SELECT country_id, last_update FROM city NATURAL JOIN country;
+SELECT country_id, last_update FROM city NATURAL LEFT JOIN country;
+SELECT country_id, last_update FROM city NATURAL LEFT JOIN country;
+SELECT country_id, last_update FROM city NATURAL RIGHT JOIN country;
+SELECT country_id, last_update FROM city NATURAL RIGHT JOIN country;
+SELECT a.country_id, a.last_update FROM city a STRAIGHT_JOIN country b ON a.country_id=b.country_id;
+SELECT a.country_id, a.last_update FROM city a STRAIGHT_JOIN country b ON a.country_id=b.country_id;
+SELECT d.deptno,d.dname,d.loc FROM scott.dept d WHERE d.deptno IN  (SELECT e.deptno FROM scott.emp e);
+SELECT d.deptno,d.dname,d.loc FROM scott.dept d WHERE d.deptno IN (SELECT e.deptno FROM scott.emp e);
+SELECT visitor_id, url FROM (SELECT id FROM log WHERE ip="123.45.67.89" order by ts desc limit 50, 10) I JOIN log ON (I.id=log.id) JOIN url ON (url.id=log.url_id) order by TS desc;
+SELECT visitor_id, url FROM (SELECT id FROM log WHERE ip="123.45.67.89" order by ts desc limit 50, 10) I JOIN log ON (I.id=log.id) JOIN url ON (url.id=log.url_id) order by TS desc;
+DELETE city, country FROM city INNER JOIN country using (country_id) WHERE city.city_id = 1;
+DELETE city, country FROM city INNER JOIN country using (country_id) WHERE city.city_id = 1;
+DELETE city FROM city LEFT JOIN country ON city.country_id = country.country_id WHERE country.country IS NULL;
+DELETE city FROM city LEFT JOIN country ON city.country_id = country.country_id WHERE country.country IS NULL;
+DELETE a1, a2 FROM city AS a1 INNER JOIN country AS a2 WHERE a1.country_id=a2.country_id;
+DELETE a1, a2 FROM city AS a1 INNER JOIN country AS a2 WHERE a1.country_id=a2.country_id;
+DELETE FROM a1, a2 USING city AS a1 INNER JOIN country AS a2 WHERE a1.country_id=a2.country_id;
+DELETE FROM a1, a2 USING city AS a1 INNER JOIN country AS a2 WHERE a1.country_id=a2.country_id;
+DELETE FROM film WHERE length > 100;
+DELETE FROM film WHERE length > 100;
+UPDATE city INNER JOIN country USING(country_id) SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.city_id=10;
+UPDATE city INNER JOIN country USING(country_id) SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.city_id=10;
+UPDATE city INNER JOIN country ON city.country_id = country.country_id INNER JOIN address ON city.city_id = address.city_id SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.city_id=10;
+UPDATE city INNER JOIN country ON city.country_id = country.country_id INNER JOIN address ON city.city_id = address.city_id SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.city_id=10;
+UPDATE city, country SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.country_id = country.country_id AND city.city_id=10;
+UPDATE city, country SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.country_id = country.country_id AND city.city_id=10;
+UPDATE film SET length = 10 WHERE language_id = 20;
+UPDATE film SET length = 10 WHERE language_id = 20;
+INSERT INTO city (country_id) SELECT country_id FROM country;
+INSERT INTO city (country_id) SELECT country_id FROM country;
+INSERT INTO city (country_id) VALUES (1),(2),(3);
+INSERT INTO city (country_id) VALUES (1),(2),(3);
+INSERT INTO city (country_id) VALUES (10);
+INSERT INTO city (country_id) VALUES (10);
+INSERT INTO city (country_id) SELECT 10 FROM DUAL;
+INSERT INTO city (country_id) SELECT 10 FROM DUAL;
+REPLACE INTO city (country_id) SELECT country_id FROM country;
+REPLACE INTO city (country_id) SELECT country_id FROM country;
+REPLACE INTO city (country_id) VALUES (1),(2),(3);
+REPLACE INTO city (country_id) VALUES (1),(2),(3);
+REPLACE INTO city (country_id) VALUES (10);
+REPLACE INTO city (country_id) VALUES (10);
+REPLACE INTO city (country_id) SELECT 10 FROM DUAL;
+REPLACE INTO city (country_id) SELECT 10 FROM DUAL;
+SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM  film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film;
+SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film;
+SELECT * FROM film WHERE language_id = (SELECT language_id FROM language LIMIT 1);
+SELECT * FROM film WHERE language_id = (SELECT language_id FROM language LIMIT 1);
+SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id;
+SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id;
+SELECT * FROM (SELECT * FROM actor WHERE last_update='2006-02-15 04:34:33' and last_name='CHASE') t WHERE last_update='2006-02-15 04:34:33' and last_name='CHASE' GROUP BY first_name;
+SELECT * FROM (SELECT * FROM actor WHERE last_update='2006-02-15 04:34:33' and last_name='CHASE') t WHERE last_update='2006-02-15 04:34:33' and last_name='CHASE' GROUP BY first_name;
+SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id;
+SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id;
+SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id WHERE o.country_id is null union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id WHERE i.city_id is null;
+SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id WHERE o.country_id is null union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id WHERE i.city_id is null;
+SELECT first_name,last_name,email FROM customer STRAIGHT_JOIN address ON customer.address_id=address.address_id;
+SELECT first_name,last_name,email FROM customer STRAIGHT_JOIN address ON customer.address_id=address.address_id;
+SELECT ID,name FROM (SELECT address FROM customer_list WHERE SID=1 order by phone limit 50,10) a JOIN customer_list l ON (a.address=l.address) JOIN city c ON (c.city=l.city) order by phone desc;
+SELECT ID,name FROM (SELECT address FROM customer_list WHERE SID=1 order by phone limit 50,10) a JOIN customer_list l ON (a.address=l.address) JOIN city c ON (c.city=l.city) order by phone desc;
+SELECT * FROM film WHERE date(last_update)='2006-02-15';
+SELECT * FROM film WHERE date(last_update)='2006-02-15';
+SELECT last_update FROM film GROUP BY date(last_update);
+SELECT last_update FROM film GROUP BY date(last_update);
+SELECT last_update FROM film order by date(last_update);
+SELECT last_update FROM film order by date(last_update);
+SELECT description FROM film WHERE description IN('NEWS','asd') GROUP BY description;
+SELECT description FROM film WHERE description IN('NEWS','asd') GROUP BY description;
+alter table address add index idx_city_id(city_id);
+alter table address add index idx_city_id(city_id);
+alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
+alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
+alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
+alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
+SELECT DATE_FORMAT(t.atm, '%Y-%m-%d'), COUNT(DISTINCT (t.usr)) FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00' AND t.agent LIKE '%Chrome%' AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d') ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
+create table hello.t (id int unsigned);
+create table hello.t (id int unsigned);
diff --git a/ast/testdata/TestGetQuotedString.golden b/ast/testdata/TestGetQuotedString.golden
new file mode 100644
index 0000000..9cf0b91
--- /dev/null
+++ b/ast/testdata/TestGetQuotedString.golden
@@ -0,0 +1,24 @@
+orignal: "hello world"
+quoted: "hello world"
+orignal: `hello world`
+quoted: `hello world`
+orignal: 'hello world'
+quoted: 'hello world'
+orignal: hello world
+quoted: 
+orignal: 'hello \'world'
+quoted: 'hello \'world'
+orignal: "hello \"wor\"ld"
+quoted: "hello \"wor\"ld"
+orignal: "hello \"world"
+quoted: "hello \"world"
+orignal: ""
+quoted: ""
+orignal: ''
+quoted: ''
+orignal: ``
+quoted: ``
+orignal: 'hello 'world'
+quoted: 'hello '
+orignal: "hello "world"
+quoted: "hello "
diff --git a/ast/testdata/TestLeftNewLines.golden b/ast/testdata/TestLeftNewLines.golden
new file mode 100644
index 0000000..e224937
--- /dev/null
+++ b/ast/testdata/TestLeftNewLines.golden
@@ -0,0 +1,3 @@
+1
+0
+0
diff --git a/ast/testdata/TestNewLines.golden b/ast/testdata/TestNewLines.golden
new file mode 100644
index 0000000..a384d6e
--- /dev/null
+++ b/ast/testdata/TestNewLines.golden
@@ -0,0 +1,3 @@
+1
+2
+0
diff --git a/ast/testdata/TestSplitStatement.golden b/ast/testdata/TestSplitStatement.golden
new file mode 100644
index 0000000..e4d6516
--- /dev/null
+++ b/ast/testdata/TestSplitStatement.golden
@@ -0,0 +1,27 @@
+0 select * from test;
+1 select 'asd;fas', col from test;
+2 -- select * from test;hello
+3 #select * from test;hello
+4 select * /*comment*/from test;
+5 select * /*comment;*/from test;
+6 select * /*comment
+		;*/
+		from test;
+7 select * from test
+8 /*comment*/
+9 /*comment*/;
+10 --
+11 -- comment
+12 # comment
+13 select
+*
+-- comment
+0 select * from test\Ghello
+1 select 'hello\Gworld', col from test\Ghello
+2 -- select * from test\Ghello
+3 #select * from test\Ghello
+4 select * /*comment*/from test\Ghello
+5 select * /*comment;*/from test\Ghello
+6 select * /*comment
+        \\G*/
+        from test\\Ghello
diff --git a/ast/testdata/TestTokenize.golden b/ast/testdata/TestTokenize.golden
new file mode 100644
index 0000000..5e3c3b7
--- /dev/null
+++ b/ast/testdata/TestTokenize.golden
@@ -0,0 +1,166 @@
+SELECT * FROM film WHERE length = 86;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 86; 0}]
+SELECT * FROM film WHERE length IS NULL;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {1 IS  0} {1 NULL; 0}]
+SELECT * FROM film HAVING title = 'abc';
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 HAVING  0} {1 title  0} {7 = 0} {0   0} {2 'abc' 0} {7 ; 0}]
+SELECT * FROM sakila.film WHERE length >= 60;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 sakila. 0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {7 = 0} {0   0} {10 60; 0}]
+SELECT * FROM sakila.film WHERE length >= '60';
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 sakila. 0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {7 = 0} {0   0} {2 '60' 0} {7 ; 0}]
+SELECT * FROM film WHERE length BETWEEN 60 AND 84;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {1 BETWEEN  0} {10 60  0} {6 AND  0} {10 84; 0}]
+SELECT * FROM film WHERE title LIKE 'AIR%';
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 title  0} {1 LIKE  0} {2 'AIR%' 0} {7 ; 0}]
+SELECT * FROM film WHERE title IS NOT NULL;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 title  0} {1 IS  0} {1 NOT  0} {1 NULL; 0}]
+SELECT * FROM film WHERE length = 114 and title = 'ALABAMA DEVIL';
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 114  0} {6 AND  0} {1 title  0} {7 = 0} {0   0} {2 'ALABAMA DEVIL' 0} {7 ; 0}]
+SELECT * FROM film WHERE length > 100 and title = 'ALABAMA DEVIL';
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {0   0} {10 100  0} {6 AND  0} {1 title  0} {7 = 0} {0   0} {2 'ALABAMA DEVIL' 0} {7 ; 0}]
+SELECT * FROM film WHERE length > 100 and language_id < 10 and title = 'xyz';
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {0   0} {10 100  0} {6 AND  0} {1 language_id  0} {7 < 0} {0   0} {10 10  0} {6 AND  0} {1 title  0} {7 = 0} {0   0} {2 'xyz' 0} {7 ; 0}]
+SELECT * FROM film WHERE length > 100 and language_id < 10;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {0   0} {10 100  0} {6 AND  0} {1 language_id  0} {7 < 0} {0   0} {10 10; 0}]
+SELECT release_year, sum(length) FROM film WHERE length = 123 AND language_id = 1 GROUP BY release_year;
+[{5 SELECT  0} {1 release_year, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 123  0} {6 AND  0} {1 language_id  0} {7 = 0} {0   0} {10 1  0} {5 GROUP BY  0} {1 release_year; 0}]
+SELECT release_year, sum(length) FROM film WHERE length >= 123 GROUP BY release_year;
+[{5 SELECT  0} {1 release_year, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {7 = 0} {0   0} {10 123  0} {5 GROUP BY  0} {1 release_year; 0}]
+SELECT release_year, language_id, sum(length) FROM film GROUP BY release_year, language_id;
+[{5 SELECT  0} {1 release_year, 0} {0   0} {1 language_id, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 GROUP BY  0} {1 release_year, 0} {0   0} {1 language_id; 0}]
+SELECT release_year, sum(length) FROM film WHERE length = 123 GROUP BY release_year,(length+language_id);
+[{5 SELECT  0} {1 release_year, 0} {0   0} {4 SUM( 0} {4 LENGTH) 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 123  0} {5 GROUP BY  0} {1 release_year, 0} {7 ( 0} {4 LENGTH+ 0} {1 language_id) 0} {7 ; 0}]
+SELECT release_year, sum(film_id) FROM film GROUP BY release_year;
+[{5 SELECT  0} {1 release_year, 0} {0   0} {4 SUM( 0} {1 film_id) 0} {0   0} {5 FROM  0} {1 film  0} {5 GROUP BY  0} {1 release_year; 0}]
+SELECT * FROM address GROUP BY address,district;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 address  0} {5 GROUP BY  0} {1 address, 0} {1 district; 0}]
+SELECT title FROM film WHERE ABS(language_id) = 3 GROUP BY title;
+[{5 SELECT  0} {1 title  0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 ABS( 0} {1 language_id) 0} {0   0} {7 = 0} {0   0} {10 3  0} {5 GROUP BY  0} {1 title; 0}]
+SELECT language_id FROM film WHERE length = 123 GROUP BY release_year ORDER BY language_id;
+[{5 SELECT  0} {1 language_id  0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 123  0} {5 GROUP BY  0} {1 release_year  0} {5 ORDER BY  0} {1 language_id; 0}]
+SELECT release_year FROM film WHERE length = 123 GROUP BY release_year ORDER BY release_year;
+[{5 SELECT  0} {1 release_year  0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 123  0} {5 GROUP BY  0} {1 release_year  0} {5 ORDER BY  0} {1 release_year; 0}]
+SELECT * FROM film WHERE length = 123 ORDER BY release_year ASC, language_id DESC;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 123  0} {5 ORDER BY  0} {1 release_year  0} {1 ASC, 0} {0   0} {1 language_id  0} {1 DESC; 0}]
+SELECT release_year FROM film WHERE length = 123 GROUP BY release_year ORDER BY release_year LIMIT 10;
+[{5 SELECT  0} {1 release_year  0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 123  0} {5 GROUP BY  0} {1 release_year  0} {5 ORDER BY  0} {1 release_year  0} {5 LIMIT  0} {10 10; 0}]
+SELECT * FROM film WHERE length = 123 ORDER BY release_year LIMIT 10;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 123  0} {5 ORDER BY  0} {1 release_year  0} {5 LIMIT  0} {10 10; 0}]
+SELECT * FROM film ORDER BY release_year LIMIT 10;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 ORDER BY  0} {1 release_year  0} {5 LIMIT  0} {10 10; 0}]
+SELECT film_id FROM film ORDER BY release_year LIMIT 10;
+[{5 SELECT  0} {1 film_id  0} {5 FROM  0} {1 film  0} {5 ORDER BY  0} {1 release_year  0} {5 LIMIT  0} {10 10; 0}]
+SELECT * FROM film WHERE length > 100 ORDER BY length LIMIT 10;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {0   0} {10 100  0} {5 ORDER BY  0} {4 LENGTH  0} {5 LIMIT  0} {10 10; 0}]
+SELECT * FROM film WHERE length < 100 ORDER BY length LIMIT 10;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 < 0} {0   0} {10 100  0} {5 ORDER BY  0} {4 LENGTH  0} {5 LIMIT  0} {10 10; 0}]
+SELECT * FROM customer WHERE address_id in (224,510) ORDER BY last_name;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 customer  0} {5 WHERE  0} {1 address_id  0} {1 in  0} {7 ( 0} {10 224, 0} {10 510) 0} {0   0} {5 ORDER BY  0} {1 last_name; 0}]
+SELECT * FROM film WHERE release_year = 2016 AND length != 1 ORDER BY title;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 release_year  0} {7 = 0} {0   0} {10 2016  0} {6 AND  0} {4 LENGTH  0} {7 ! 0} {7 = 0} {0   0} {10 1  0} {5 ORDER BY  0} {1 title; 0}]
+SELECT title FROM film WHERE release_year = 1995;
+[{5 SELECT  0} {1 title  0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 release_year  0} {7 = 0} {0   0} {10 1995; 0}]
+SELECT title, replacement_cost FROM film WHERE language_id = 5 AND length = 70;
+[{5 SELECT  0} {1 title, 0} {0   0} {1 replacement_cost  0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 language_id  0} {7 = 0} {0   0} {10 5  0} {6 AND  0} {4 LENGTH  0} {7 = 0} {0   0} {10 70; 0}]
+SELECT title FROM film WHERE language_id > 5 AND length > 70;
+[{5 SELECT  0} {1 title  0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 language_id  0} {7 > 0} {0   0} {10 5  0} {6 AND  0} {4 LENGTH  0} {7 > 0} {0   0} {10 70; 0}]
+SELECT * FROM film WHERE length = 100 and title = 'xyz' ORDER BY release_year;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 = 0} {0   0} {10 100  0} {6 AND  0} {1 title  0} {7 = 0} {0   0} {2 'xyz' 0} {0   0} {5 ORDER BY  0} {1 release_year; 0}]
+SELECT * FROM film WHERE length > 100 and title = 'xyz' ORDER BY release_year;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {0   0} {10 100  0} {6 AND  0} {1 title  0} {7 = 0} {0   0} {2 'xyz' 0} {0   0} {5 ORDER BY  0} {1 release_year; 0}]
+SELECT * FROM film WHERE length > 100 ORDER BY release_year;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {0   0} {10 100  0} {5 ORDER BY  0} {1 release_year; 0}]
+SELECT * FROM city a INNER JOIN country b ON a.country_id=b.country_id;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 a  0} {6 INNER JOIN  0} {1 country  0} {1 b  0} {1 ON  0} {1 a. 0} {1 country_id= 0} {1 b. 0} {1 country_id; 0}]
+SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 a  0} {6 LEFT JOIN  0} {1 country  0} {1 b  0} {1 ON  0} {1 a. 0} {1 country_id= 0} {1 b. 0} {1 country_id; 0}]
+SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 a  0} {6 RIGHT JOIN  0} {1 country  0} {1 b  0} {1 ON  0} {1 a. 0} {1 country_id= 0} {1 b. 0} {1 country_id; 0}]
+SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id WHERE b.last_update IS NULL;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 a  0} {6 LEFT JOIN  0} {1 country  0} {1 b  0} {1 ON  0} {1 a. 0} {1 country_id= 0} {1 b. 0} {1 country_id  0} {5 WHERE  0} {1 b. 0} {1 last_update  0} {1 IS  0} {1 NULL; 0}]
+SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id WHERE a.last_update IS NULL;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 a  0} {6 RIGHT JOIN  0} {1 country  0} {1 b  0} {1 ON  0} {1 a. 0} {1 country_id= 0} {1 b. 0} {1 country_id  0} {5 WHERE  0} {1 a. 0} {1 last_update  0} {1 IS  0} {1 NULL; 0}]
+SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id UNION SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 a  0} {6 LEFT JOIN  0} {1 country  0} {1 b  0} {1 ON  0} {1 a. 0} {1 country_id= 0} {1 b. 0} {1 country_id  0} {5 UNION  0} {5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 a  0} {6 RIGHT JOIN  0} {1 country  0} {1 b  0} {1 ON  0} {1 a. 0} {1 country_id= 0} {1 b. 0} {1 country_id; 0}]
+SELECT * FROM city a RIGHT JOIN country b ON a.country_id=b.country_id WHERE a.last_update IS NULL UNION SELECT * FROM city a LEFT JOIN country b ON a.country_id=b.country_id WHERE b.last_update IS NULL;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 a  0} {6 RIGHT JOIN  0} {1 country  0} {1 b  0} {1 ON  0} {1 a. 0} {1 country_id= 0} {1 b. 0} {1 country_id  0} {5 WHERE  0} {1 a. 0} {1 last_update  0} {1 IS  0} {1 NULL  0} {5 UNION  0} {5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 a  0} {6 LEFT JOIN  0} {1 country  0} {1 b  0} {1 ON  0} {1 a. 0} {1 country_id= 0} {1 b. 0} {1 country_id  0} {5 WHERE  0} {1 b. 0} {1 last_update  0} {1 IS  0} {1 NULL; 0}]
+SELECT country_id, last_update FROM city NATURAL JOIN country;
+[{5 SELECT  0} {1 country_id, 0} {0   0} {1 last_update  0} {5 FROM  0} {1 city  0} {1 NATURAL  0} {6 JOIN  0} {1 country; 0}]
+SELECT country_id, last_update FROM city NATURAL LEFT JOIN country;
+[{5 SELECT  0} {1 country_id, 0} {0   0} {1 last_update  0} {5 FROM  0} {1 city  0} {1 NATURAL  0} {6 LEFT JOIN  0} {1 country; 0}]
+SELECT country_id, last_update FROM city NATURAL RIGHT JOIN country;
+[{5 SELECT  0} {1 country_id, 0} {0   0} {1 last_update  0} {5 FROM  0} {1 city  0} {1 NATURAL  0} {6 RIGHT JOIN  0} {1 country; 0}]
+SELECT a.country_id, a.last_update FROM city a STRAIGHT_JOIN country b ON a.country_id=b.country_id;
+[{5 SELECT  0} {1 a. 0} {1 country_id, 0} {0   0} {1 a. 0} {1 last_update  0} {5 FROM  0} {1 city  0} {1 a  0} {1 STRAIGHT_JOIN  0} {1 country  0} {1 b  0} {1 ON  0} {1 a. 0} {1 country_id= 0} {1 b. 0} {1 country_id; 0}]
+SELECT d.deptno,d.dname,d.loc FROM scott.dept d WHERE d.deptno IN  (SELECT e.deptno FROM scott.emp e);
+[{5 SELECT  0} {1 d. 0} {1 deptno, 0} {1 d. 0} {1 dname, 0} {1 d. 0} {1 loc  0} {5 FROM  0} {1 scott. 0} {1 dept  0} {1 d  0} {5 WHERE  0} {1 d. 0} {1 deptno  0} {1 IN  0} {0   0} {7 ( 0} {5 SELECT  0} {1 e. 0} {1 deptno  0} {5 FROM  0} {1 scott. 0} {1 emp  0} {1 e) 0} {7 ; 0}]
+SELECT visitor_id, url FROM (SELECT id FROM log WHERE ip="123.45.67.89" order by ts desc limit 50, 10) I JOIN log ON (I.id=log.id) JOIN url ON (url.id=log.url_id) order by TS desc;
+[{5 SELECT  0} {1 visitor_id, 0} {0   0} {1 url  0} {5 FROM  0} {7 ( 0} {5 SELECT  0} {1 id  0} {5 FROM  0} {4 LOG  0} {5 WHERE  0} {1 ip= 0} {2 "123.45.67.89" 0} {0   0} {5 ORDER BY  0} {1 ts  0} {1 desc  0} {5 LIMIT  0} {10 50, 0} {0   0} {10 10) 0} {0   0} {1 I  0} {6 JOIN  0} {4 LOG  0} {1 ON  0} {7 ( 0} {1 I. 0} {1 id= 0} {4 LOG. 0} {1 id) 0} {0   0} {6 JOIN  0} {1 url  0} {1 ON  0} {7 ( 0} {1 url. 0} {1 id= 0} {4 LOG. 0} {1 url_id) 0} {0   0} {5 ORDER BY  0} {1 TS  0} {1 desc; 0}]
+DELETE city, country FROM city INNER JOIN country using (country_id) WHERE city.city_id = 1;
+[{1 DELETE  0} {1 city, 0} {0   0} {1 country  0} {5 FROM  0} {1 city  0} {6 INNER JOIN  0} {1 country  0} {1 using  0} {7 ( 0} {1 country_id) 0} {0   0} {5 WHERE  0} {1 city. 0} {1 city_id  0} {7 = 0} {0   0} {10 1; 0}]
+DELETE city FROM city LEFT JOIN country ON city.country_id = country.country_id WHERE country.country IS NULL;
+[{1 DELETE  0} {1 city  0} {5 FROM  0} {1 city  0} {6 LEFT JOIN  0} {1 country  0} {1 ON  0} {1 city. 0} {1 country_id  0} {7 = 0} {0   0} {1 country. 0} {1 country_id  0} {5 WHERE  0} {1 country. 0} {1 country  0} {1 IS  0} {1 NULL; 0}]
+DELETE a1, a2 FROM city AS a1 INNER JOIN country AS a2 WHERE a1.country_id=a2.country_id;
+[{1 DELETE  0} {1 a1, 0} {0   0} {1 a2  0} {5 FROM  0} {1 city  0} {1 AS  0} {1 a1  0} {6 INNER JOIN  0} {1 country  0} {1 AS  0} {1 a2  0} {5 WHERE  0} {1 a1. 0} {1 country_id= 0} {1 a2. 0} {1 country_id; 0}]
+DELETE FROM a1, a2 USING city AS a1 INNER JOIN country AS a2 WHERE a1.country_id=a2.country_id;
+[{5 DELETE FROM  0} {1 a1, 0} {0   0} {1 a2  0} {1 USING  0} {1 city  0} {1 AS  0} {1 a1  0} {6 INNER JOIN  0} {1 country  0} {1 AS  0} {1 a2  0} {5 WHERE  0} {1 a1. 0} {1 country_id= 0} {1 a2. 0} {1 country_id; 0}]
+DELETE FROM film WHERE length > 100;
+[{5 DELETE FROM  0} {1 film  0} {5 WHERE  0} {4 LENGTH  0} {7 > 0} {0   0} {10 100; 0}]
+UPDATE city INNER JOIN country USING(country_id) SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.city_id=10;
+[{5 UPDATE  0} {1 city  0} {6 INNER JOIN  0} {1 country  0} {1 USING( 0} {1 country_id) 0} {0   0} {5 SET  0} {1 city. 0} {1 city  0} {7 = 0} {0   0} {2 'Abha' 0} {7 , 0} {0   0} {1 city. 0} {1 last_update  0} {7 = 0} {0   0} {2 '2006-02-15 04:45:25' 0} {7 , 0} {0   0} {1 country. 0} {1 country  0} {7 = 0} {0   0} {2 'Afghanistan' 0} {0   0} {5 WHERE  0} {1 city. 0} {1 city_id= 0} {10 10; 0}]
+UPDATE city INNER JOIN country ON city.country_id = country.country_id INNER JOIN address ON city.city_id = address.city_id SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.city_id=10;
+[{5 UPDATE  0} {1 city  0} {6 INNER JOIN  0} {1 country  0} {1 ON  0} {1 city. 0} {1 country_id  0} {7 = 0} {0   0} {1 country. 0} {1 country_id  0} {6 INNER JOIN  0} {1 address  0} {1 ON  0} {1 city. 0} {1 city_id  0} {7 = 0} {0   0} {1 address. 0} {1 city_id  0} {5 SET  0} {1 city. 0} {1 city  0} {7 = 0} {0   0} {2 'Abha' 0} {7 , 0} {0   0} {1 city. 0} {1 last_update  0} {7 = 0} {0   0} {2 '2006-02-15 04:45:25' 0} {7 , 0} {0   0} {1 country. 0} {1 country  0} {7 = 0} {0   0} {2 'Afghanistan' 0} {0   0} {5 WHERE  0} {1 city. 0} {1 city_id= 0} {10 10; 0}]
+UPDATE city, country SET city.city = 'Abha', city.last_update = '2006-02-15 04:45:25', country.country = 'Afghanistan' WHERE city.country_id = country.country_id AND city.city_id=10;
+[{5 UPDATE  0} {1 city, 0} {0   0} {1 country  0} {5 SET  0} {1 city. 0} {1 city  0} {7 = 0} {0   0} {2 'Abha' 0} {7 , 0} {0   0} {1 city. 0} {1 last_update  0} {7 = 0} {0   0} {2 '2006-02-15 04:45:25' 0} {7 , 0} {0   0} {1 country. 0} {1 country  0} {7 = 0} {0   0} {2 'Afghanistan' 0} {0   0} {5 WHERE  0} {1 city. 0} {1 country_id  0} {7 = 0} {0   0} {1 country. 0} {1 country_id  0} {6 AND  0} {1 city. 0} {1 city_id= 0} {10 10; 0}]
+UPDATE film SET length = 10 WHERE language_id = 20;
+[{5 UPDATE  0} {1 film  0} {5 SET  0} {4 LENGTH  0} {7 = 0} {0   0} {10 10  0} {5 WHERE  0} {1 language_id  0} {7 = 0} {0   0} {10 20; 0}]
+INSERT INTO city (country_id) SELECT country_id FROM country;
+[{4 INSERT  0} {1 INTO  0} {1 city  0} {7 ( 0} {1 country_id) 0} {0   0} {5 SELECT  0} {1 country_id  0} {5 FROM  0} {1 country; 0}]
+INSERT INTO city (country_id) VALUES (1),(2),(3);
+[{4 INSERT  0} {1 INTO  0} {1 city  0} {7 ( 0} {1 country_id) 0} {0   0} {5 VALUES  0} {7 ( 0} {10 1) 0} {7 , 0} {7 ( 0} {10 2) 0} {7 , 0} {7 ( 0} {10 3) 0} {7 ; 0}]
+INSERT INTO city (country_id) VALUES (10);
+[{4 INSERT  0} {1 INTO  0} {1 city  0} {7 ( 0} {1 country_id) 0} {0   0} {5 VALUES  0} {7 ( 0} {10 10) 0} {7 ; 0}]
+INSERT INTO city (country_id) SELECT 10 FROM DUAL;
+[{4 INSERT  0} {1 INTO  0} {1 city  0} {7 ( 0} {1 country_id) 0} {0   0} {5 SELECT  0} {10 10  0} {5 FROM  0} {1 DUAL; 0}]
+REPLACE INTO city (country_id) SELECT country_id FROM country;
+[{4 REPLACE  0} {1 INTO  0} {1 city  0} {7 ( 0} {1 country_id) 0} {0   0} {5 SELECT  0} {1 country_id  0} {5 FROM  0} {1 country; 0}]
+REPLACE INTO city (country_id) VALUES (1),(2),(3);
+[{4 REPLACE  0} {1 INTO  0} {1 city  0} {7 ( 0} {1 country_id) 0} {0   0} {5 VALUES  0} {7 ( 0} {10 1) 0} {7 , 0} {7 ( 0} {10 2) 0} {7 , 0} {7 ( 0} {10 3) 0} {7 ; 0}]
+REPLACE INTO city (country_id) VALUES (10);
+[{4 REPLACE  0} {1 INTO  0} {1 city  0} {7 ( 0} {1 country_id) 0} {0   0} {5 VALUES  0} {7 ( 0} {10 10) 0} {7 ; 0}]
+REPLACE INTO city (country_id) SELECT 10 FROM DUAL;
+[{4 REPLACE  0} {1 INTO  0} {1 city  0} {7 ( 0} {1 country_id) 0} {0   0} {5 SELECT  0} {10 10  0} {5 FROM  0} {1 DUAL; 0}]
+SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM ( SELECT film_id FROM  film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film ) film;
+[{5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {7 ( 0} {0   0} {5 SELECT  0} {1 film_id  0} {5 FROM  0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film  0} {7 ) 0} {0   0} {1 film; 0}]
+SELECT * FROM film WHERE language_id = (SELECT language_id FROM language LIMIT 1);
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 language_id  0} {7 = 0} {0   0} {7 ( 0} {5 SELECT  0} {1 language_id  0} {5 FROM  0} {1 language  0} {5 LIMIT  0} {10 1) 0} {7 ; 0}]
+SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 i  0} {6 LEFT JOIN  0} {1 country  0} {1 o  0} {1 ON  0} {1 i. 0} {1 city_id= 0} {1 o. 0} {1 country_id  0} {5 UNION  0} {5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 i  0} {6 RIGHT JOIN  0} {1 country  0} {1 o  0} {1 ON  0} {1 i. 0} {1 city_id= 0} {1 o. 0} {1 country_id; 0}]
+SELECT * FROM (SELECT * FROM actor WHERE last_update='2006-02-15 04:34:33' and last_name='CHASE') t WHERE last_update='2006-02-15 04:34:33' and last_name='CHASE' GROUP BY first_name;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {7 ( 0} {5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 actor  0} {5 WHERE  0} {1 last_update= 0} {2 '2006-02-15 04:34:33' 0} {0   0} {6 AND  0} {1 last_name= 0} {2 'CHASE' 0} {7 ) 0} {0   0} {1 t  0} {5 WHERE  0} {1 last_update= 0} {2 '2006-02-15 04:34:33' 0} {0   0} {6 AND  0} {1 last_name= 0} {2 'CHASE' 0} {0   0} {5 GROUP BY  0} {1 first_name; 0}]
+SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 i  0} {6 LEFT JOIN  0} {1 country  0} {1 o  0} {1 ON  0} {1 i. 0} {1 city_id= 0} {1 o. 0} {1 country_id  0} {5 UNION  0} {5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 i  0} {6 RIGHT JOIN  0} {1 country  0} {1 o  0} {1 ON  0} {1 i. 0} {1 city_id= 0} {1 o. 0} {1 country_id; 0}]
+SELECT * FROM city i left JOIN country o ON i.city_id=o.country_id WHERE o.country_id is null union SELECT * FROM city i right JOIN country o ON i.city_id=o.country_id WHERE i.city_id is null;
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 i  0} {6 LEFT JOIN  0} {1 country  0} {1 o  0} {1 ON  0} {1 i. 0} {1 city_id= 0} {1 o. 0} {1 country_id  0} {5 WHERE  0} {1 o. 0} {1 country_id  0} {1 is  0} {1 null  0} {5 UNION  0} {5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 city  0} {1 i  0} {6 RIGHT JOIN  0} {1 country  0} {1 o  0} {1 ON  0} {1 i. 0} {1 city_id= 0} {1 o. 0} {1 country_id  0} {5 WHERE  0} {1 i. 0} {1 city_id  0} {1 is  0} {1 null; 0}]
+SELECT first_name,last_name,email FROM customer STRAIGHT_JOIN address ON customer.address_id=address.address_id;
+[{5 SELECT  0} {1 first_name, 0} {1 last_name, 0} {1 email  0} {5 FROM  0} {1 customer  0} {1 STRAIGHT_JOIN  0} {1 address  0} {1 ON  0} {1 customer. 0} {1 address_id= 0} {1 address. 0} {1 address_id; 0}]
+SELECT ID,name FROM (SELECT address FROM customer_list WHERE SID=1 order by phone limit 50,10) a JOIN customer_list l ON (a.address=l.address) JOIN city c ON (c.city=l.city) order by phone desc;
+[{5 SELECT  0} {1 ID, 0} {1 name  0} {5 FROM  0} {7 ( 0} {5 SELECT  0} {1 address  0} {5 FROM  0} {1 customer_list  0} {5 WHERE  0} {1 SID= 0} {10 1  0} {5 ORDER BY  0} {1 phone  0} {5 LIMIT  0} {10 50, 0} {10 10) 0} {0   0} {1 a  0} {6 JOIN  0} {1 customer_list  0} {1 l  0} {1 ON  0} {7 ( 0} {1 a. 0} {1 address= 0} {1 l. 0} {1 address) 0} {0   0} {6 JOIN  0} {1 city  0} {1 c  0} {1 ON  0} {7 ( 0} {1 c. 0} {1 city= 0} {1 l. 0} {1 city) 0} {0   0} {5 ORDER BY  0} {1 phone  0} {1 desc; 0}]
+SELECT * FROM film WHERE date(last_update)='2006-02-15';
+[{5 SELECT  0} {7 * 0} {0   0} {5 FROM  0} {1 film  0} {5 WHERE  0} {4 DATE( 0} {1 last_update) 0} {7 = 0} {2 '2006-02-15' 0} {7 ; 0}]
+SELECT last_update FROM film GROUP BY date(last_update);
+[{5 SELECT  0} {1 last_update  0} {5 FROM  0} {1 film  0} {5 GROUP BY  0} {4 DATE( 0} {1 last_update) 0} {7 ; 0}]
+SELECT last_update FROM film order by date(last_update);
+[{5 SELECT  0} {1 last_update  0} {5 FROM  0} {1 film  0} {5 ORDER BY  0} {4 DATE( 0} {1 last_update) 0} {7 ; 0}]
+SELECT description FROM film WHERE description IN('NEWS','asd') GROUP BY description;
+[{5 SELECT  0} {1 description  0} {5 FROM  0} {1 film  0} {5 WHERE  0} {1 description  0} {1 IN( 0} {2 'NEWS' 0} {7 , 0} {2 'asd' 0} {7 ) 0} {0   0} {5 GROUP BY  0} {1 description; 0}]
+alter table address add index idx_city_id(city_id);
+[{5 ALTER TABLE  0} {1 address  0} {5 ADD  0} {1 index  0} {1 idx_city_id( 0} {1 city_id) 0} {7 ; 0}]
+alter table inventory add index `idx_store_film` (`store_id`,`film_id`);
+[{5 ALTER TABLE  0} {1 inventory  0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 ; 0}]
+alter table inventory add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`),add index `idx_store_film` (`store_id`,`film_id`);
+[{5 ALTER TABLE  0} {1 inventory  0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 , 0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 , 0} {5 ADD  0} {1 index  0} {3 `idx_store_film` 0} {0   0} {7 ( 0} {3 `store_id` 0} {7 , 0} {3 `film_id` 0} {7 ) 0} {7 ; 0}]
+SELECT	DATE_FORMAT(t.atm, '%Y-%m-%d'),	COUNT(DISTINCT (t.usr))	FROM usr_terminal t WHERE t.atm > '2018-10-22 00:00:00'	AND t.agent LIKE '%Chrome%'	AND t.system = 'eip' GROUP BY DATE_FORMAT(t.atm, '%Y-%m-%d')	ORDER BY DATE_FORMAT(t.atm, '%Y-%m-%d')
+[{5 SELECT	 0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {7 , 0} {0   0} {4 COUNT( 0} {1 DISTINCT  0} {7 ( 0} {1 t. 0} {1 usr) 0} {7 ) 0} {0   0} {5 FROM  0} {1 usr_terminal  0} {1 t  0} {5 WHERE  0} {1 t. 0} {1 atm  0} {7 > 0} {0   0} {2 '2018-10-22 00:00:00' 0} {0   0} {6 AND  0} {1 t. 0} {1 agent  0} {1 LIKE  0} {2 '%Chrome%' 0} {0   0} {6 AND  0} {1 t. 0} {1 system  0} {7 = 0} {0   0} {2 'eip' 0} {0   0} {5 GROUP BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0} {0   0} {5 ORDER BY  0} {4 DATE_FORMAT( 0} {1 t. 0} {1 atm, 0} {0   0} {2 '%Y-%m-%d' 0} {7 ) 0}]
+create table hello.t (id int unsigned);
+[{1 create  0} {1 table  0} {1 hello. 0} {1 t  0} {7 ( 0} {1 id  0} {1 int  0} {1 unsigned) 0} {7 ; 0}]
diff --git a/ast/testdata/TestTokenizer.golden b/ast/testdata/TestTokenizer.golden
new file mode 100644
index 0000000..1b4d7a1
--- /dev/null
+++ b/ast/testdata/TestTokenizer.golden
@@ -0,0 +1,281 @@
+[]ast.Token{
+    {Type:57348, Val:"select", i:0},
+    {Type:57396, Val:"c1", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57396, Val:"c2", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57396, Val:"c3", i:0},
+    {Type:57353, Val:"from", i:0},
+    {Type:57396, Val:"t1", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57396, Val:"t2", i:0},
+    {Type:57384, Val:"join", i:0},
+    {Type:57396, Val:"t3", i:0},
+    {Type:57394, Val:"on", i:0},
+    {Type:57396, Val:"t1", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"c1", i:0},
+    {Type:61, Val:"=", i:0},
+    {Type:57396, Val:"t2", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"c1", i:0},
+    {Type:57412, Val:"and", i:0},
+    {Type:57396, Val:"t1", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"c3", i:0},
+    {Type:61, Val:"=", i:0},
+    {Type:57396, Val:"t3", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"c1", i:0},
+    {Type:57354, Val:"where", i:0},
+    {Type:57396, Val:"id", i:0},
+    {Type:62, Val:">", i:0},
+    {Type:57399, Val:"1000", i:0},
+}
+[]ast.Token{
+    {Type:57348, Val:"select", i:0},
+    {Type:57396, Val:"sourcetable", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57453, Val:"if", i:0},
+    {Type:40, Val:"(", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"lastcontent", i:0},
+    {Type:61, Val:"=", i:0},
+    {Type:57402, Val:":v1", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"lastupdate", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"lastcontent", i:0},
+    {Type:41, Val:")", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"lastactivity", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"totalcount", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"activity", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57396, Val:"type", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"class", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"type", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:40, Val:"(", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"nodeoptions", i:0},
+    {Type:38, Val:"&", i:0},
+    {Type:57402, Val:":v2", i:0},
+    {Type:41, Val:")", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"nounsubscribe", i:0},
+    {Type:57353, Val:"from", i:0},
+    {Type:57396, Val:"node", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:57388, Val:"inner", i:0},
+    {Type:57384, Val:"join", i:0},
+    {Type:57396, Val:"contenttype", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"type", i:0},
+    {Type:57394, Val:"on", i:0},
+    {Type:57396, Val:"type", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"contenttypeid", i:0},
+    {Type:61, Val:"=", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"contenttypeid", i:0},
+    {Type:57388, Val:"inner", i:0},
+    {Type:57384, Val:"join", i:0},
+    {Type:57396, Val:"subscribed", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"sd", i:0},
+    {Type:57394, Val:"on", i:0},
+    {Type:57396, Val:"sd", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"did", i:0},
+    {Type:61, Val:"=", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"nodeid", i:0},
+    {Type:57412, Val:"and", i:0},
+    {Type:57396, Val:"sd", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"userid", i:0},
+    {Type:61, Val:"=", i:0},
+    {Type:57402, Val:":v3", i:0},
+    {Type:57347, Val:"union", i:0},
+    {Type:57362, Val:"all", i:0},
+    {Type:57348, Val:"select", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"name", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"title", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"userid", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"keyval", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57402, Val:":v4", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"sourcetable", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57396, Val:"ifnull", i:0},
+    {Type:40, Val:"(", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"lastpost", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"joindate", i:0},
+    {Type:41, Val:")", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"lastactivity", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"posts", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"activity", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57402, Val:":v5", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"type", i:0},
+    {Type:44, Val:",", i:0},
+    {Type:57402, Val:":v6", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"nounsubscribe", i:0},
+    {Type:57353, Val:"from", i:0},
+    {Type:57396, Val:"user", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:57388, Val:"inner", i:0},
+    {Type:57384, Val:"join", i:0},
+    {Type:57396, Val:"userlist", i:0},
+    {Type:57364, Val:"as", i:0},
+    {Type:57396, Val:"ul", i:0},
+    {Type:57394, Val:"on", i:0},
+    {Type:57396, Val:"ul", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"relationid", i:0},
+    {Type:61, Val:"=", i:0},
+    {Type:57396, Val:"f", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"userid", i:0},
+    {Type:57412, Val:"and", i:0},
+    {Type:57396, Val:"ul", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"userid", i:0},
+    {Type:61, Val:"=", i:0},
+    {Type:57402, Val:":v7", i:0},
+    {Type:57354, Val:"where", i:0},
+    {Type:57396, Val:"ul", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"type", i:0},
+    {Type:61, Val:"=", i:0},
+    {Type:57402, Val:":v8", i:0},
+    {Type:57412, Val:"and", i:0},
+    {Type:57396, Val:"ul", i:0},
+    {Type:46, Val:".", i:0},
+    {Type:57396, Val:"aq", i:0},
+    {Type:61, Val:"=", i:0},
+    {Type:57402, Val:":v9", i:0},
+    {Type:57357, Val:"order", i:0},
+    {Type:57358, Val:"by", i:0},
+    {Type:57396, Val:"title", i:0},
+    {Type:57359, Val:"limit", i:0},
+    {Type:57402, Val:":v10", i:0},
+}
+[]ast.Token{
+    {Type:57348, Val:"select", i:0},
+    {Type:57396, Val:"c1", i:0},
+    {Type:57353, Val:"from", i:0},
+    {Type:57396, Val:"t1", i:0},
+    {Type:57354, Val:"where", i:0},
+    {Type:57396, Val:"id", i:0},
+    {Type:57421, Val:">=", i:0},
+    {Type:57399, Val:"1000", i:0},
+}
+[]ast.Token{
+    {Type:57348, Val:"select", i:0},
+    {Type:57589, Val:"sql_calc_found_rows", i:0},
+    {Type:57396, Val:"col", i:0},
+    {Type:57353, Val:"from", i:0},
+    {Type:57396, Val:"tbl", i:0},
+    {Type:57354, Val:"where", i:0},
+    {Type:57396, Val:"id", i:0},
+    {Type:62, Val:">", i:0},
+    {Type:57399, Val:"1000", i:0},
+}
+[]ast.Token{
+    {Type:57348, Val:"select", i:0},
+    {Type:42, Val:"*", i:0},
+    {Type:57353, Val:"from", i:0},
+    {Type:57396, Val:"tb", i:0},
+    {Type:57354, Val:"where", i:0},
+    {Type:57396, Val:"id", i:0},
+    {Type:61, Val:"=", i:0},
+    {Type:57402, Val:":v1", i:0},
+    {Type:59, Val:";", i:0},
+}
+[]ast.Token{
+    {Type:57348, Val:"select", i:0},
+    {Type:42, Val:"*", i:0},
+    {Type:57353, Val:"from", i:0},
+    {Type:57396, Val:"tb", i:0},
+    {Type:57354, Val:"where", i:0},
+    {Type:57396, Val:"id", i:0},
+    {Type:57424, Val:"is", i:0},
+    {Type:57407, Val:"null", i:0},
+    {Type:59, Val:";", i:0},
+}
+[]ast.Token{
+    {Type:57348, Val:"select", i:0},
+    {Type:42, Val:"*", i:0},
+    {Type:57353, Val:"from", i:0},
+    {Type:57396, Val:"tb", i:0},
+    {Type:57354, Val:"where", i:0},
+    {Type:57396, Val:"id", i:0},
+    {Type:57424, Val:"is", i:0},
+    {Type:57413, Val:"not", i:0},
+    {Type:57407, Val:"null", i:0},
+    {Type:59, Val:";", i:0},
+}
+[]ast.Token{
+    {Type:57348, Val:"select", i:0},
+    {Type:42, Val:"*", i:0},
+    {Type:57353, Val:"from", i:0},
+    {Type:57396, Val:"tb", i:0},
+    {Type:57354, Val:"where", i:0},
+    {Type:57396, Val:"id", i:0},
+    {Type:57414, Val:"between", i:0},
+    {Type:57399, Val:"1", i:0},
+    {Type:57412, Val:"and", i:0},
+    {Type:57399, Val:"3", i:0},
+    {Type:59, Val:";", i:0},
+}
+[]ast.Token{
+    {Type:57441, Val:"alter", i:0},
+    {Type:57448, Val:"table", i:0},
+    {Type:57396, Val:"inventory", i:0},
+    {Type:57445, Val:"add", i:0},
+    {Type:57449, Val:"index", i:0},
+    {Type:57396, Val:"idx_store_film", i:0},
+    {Type:57396, Val:" (", i:0},
+    {Type:57396, Val:"store_id", i:0},
+    {Type:57396, Val:",", i:0},
+    {Type:57396, Val:"film_id", i:0},
+    {Type:57346, Val:");", i:0},
+}
diff --git a/ast/token_test.go b/ast/token_test.go
index 197dfe8..8bb79dd 100644
--- a/ast/token_test.go
+++ b/ast/token_test.go
@@ -25,7 +25,19 @@ import (
 	"github.com/kr/pretty"
 )
 
-func TestTokenizer(_ *testing.T) {
+func TestTokenize(t *testing.T) {
+	err := common.GoldenDiff(func() {
+		for _, sql := range common.TestSQLs {
+			fmt.Println(sql)
+			fmt.Println(Tokenize(sql))
+		}
+	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
+	}
+}
+
+func TestTokenizer(t *testing.T) {
 	sqls := []string{
 		"select c1,c2,c3 from t1,t2 join t3 on t1.c1=t2.c1 and t1.c3=t3.c1 where id>1000",
 		"select sourcetable, if(f.lastcontent = ?, f.lastupdate, f.lastcontent) as lastactivity, f.totalcount as activity, type.class as type, (f.nodeoptions & ?) as nounsubscribe from node as f inner join contenttype as type on type.contenttypeid = f.contenttypeid inner join subscribed as sd on sd.did = f.nodeid and sd.userid = ? union all select f.name as title, f.userid as keyval, ? as sourcetable, ifnull(f.lastpost, f.joindate) as lastactivity, f.posts as activity, ? as type, ? as nounsubscribe from user as f inner join userlist as ul on ul.relationid = f.userid and ul.userid = ? where ul.type = ? and ul.aq = ? order by title limit ?",
@@ -37,8 +49,13 @@ func TestTokenizer(_ *testing.T) {
 		"SELECT * FROM tb WHERE id between 1 and 3;",
 		"alter table inventory add index idx_store_film` (`store_id`,`film_id`);",
 	}
-	for _, sql := range sqls {
-		pretty.Println(Tokenizer(sql))
+	err := common.GoldenDiff(func() {
+		for _, sql := range sqls {
+			pretty.Println(Tokenizer(sql))
+		}
+	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
 	}
 }
 
@@ -57,23 +74,27 @@ func TestGetQuotedString(t *testing.T) {
 		`'hello 'world'`,
 		`"hello "world"`,
 	}
-	for _, s := range str {
-		fmt.Printf("orignal: %s\nquoted: %s\n", s, getQuotedString(s))
-	}
-}
-
-func TestTokenizer2(t *testing.T) {
-	for _, sql := range common.TestSQLs {
-		fmt.Println(sql)
-		fmt.Println(Tokenize(sql))
+	err := common.GoldenDiff(func() {
+		for _, s := range str {
+			fmt.Printf("orignal: %s\nquoted: %s\n", s, getQuotedString(s))
+		}
+	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
 	}
 }
 
 func TestCompress(t *testing.T) {
-	for _, sql := range common.TestSQLs {
-		fmt.Println(sql)
-		fmt.Println(Compress(sql))
+	err := common.GoldenDiff(func() {
+		for _, sql := range common.TestSQLs {
+			fmt.Println(sql)
+			fmt.Println(Compress(sql))
+		}
+	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
 	}
+
 }
 
 func TestFormat(t *testing.T) {
@@ -97,8 +118,8 @@ func TestSplitStatement(t *testing.T) {
 		[]byte("select * /*comment*/from test;hello"),
 		[]byte("select * /*comment;*/from test;hello"),
 		[]byte(`select * /*comment
-        ;*/
-        from test;hello`),
+		;*/
+		from test;hello`),
 		[]byte(`select * from test`),
 		// https://github.com/XiaoMi/soar/issues/66
 		[]byte(`/*comment*/`),
@@ -106,9 +127,11 @@ func TestSplitStatement(t *testing.T) {
 		[]byte(`--`),
 		[]byte(`-- comment`),
 		[]byte(`# comment`),
-	}
-	for _, buf := range bufs {
-		fmt.Println(SplitStatement(buf, []byte(common.Config.Delimiter)))
+		[]byte(`select
+*
+-- comment
+from tb
+where col = 1`),
 	}
 	buf2s := [][]byte{
 		[]byte("select * from test\\Ghello"),
@@ -121,8 +144,18 @@ func TestSplitStatement(t *testing.T) {
         \\G*/
         from test\\Ghello`),
 	}
-	for _, buf := range buf2s {
-		fmt.Println(SplitStatement(buf, []byte("\\G")))
+	err := common.GoldenDiff(func() {
+		for i, buf := range bufs {
+			sql, _, _ := SplitStatement(buf, []byte(common.Config.Delimiter))
+			fmt.Println(i, sql)
+		}
+		for i, buf := range buf2s {
+			sql, _, _ := SplitStatement(buf, []byte(common.Config.Delimiter))
+			fmt.Println(i, sql)
+		}
+	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
 	}
 }
 
@@ -135,8 +168,13 @@ func TestLeftNewLines(t *testing.T) {
         from test;hello`),
 		[]byte(`select * from test`),
 	}
-	for _, buf := range bufs {
-		fmt.Println(LeftNewLines(buf))
+	err := common.GoldenDiff(func() {
+		for _, buf := range bufs {
+			fmt.Println(LeftNewLines(buf))
+		}
+	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
 	}
 }
 
@@ -149,7 +187,12 @@ func TestNewLines(t *testing.T) {
         from test;hello`),
 		[]byte(`select * from test`),
 	}
-	for _, buf := range bufs {
-		fmt.Println(NewLines(buf))
+	err := common.GoldenDiff(func() {
+		for _, buf := range bufs {
+			fmt.Println(NewLines(buf))
+		}
+	}, t.Name(), update)
+	if nil != err {
+		t.Fatal(err)
 	}
 }
diff --git a/database/explain.go b/database/explain.go
index 250ecec..2924740 100644
--- a/database/explain.go
+++ b/database/explain.go
@@ -28,6 +28,7 @@ import (
 	"github.com/XiaoMi/soar/ast"
 	"github.com/XiaoMi/soar/common"
 
+	tidb "github.com/pingcap/tidb/ast"
 	"github.com/tidwall/gjson"
 	"vitess.io/vitess/go/vt/sqlparser"
 )
@@ -502,8 +503,32 @@ func (db *Connector) supportExplainWrite() (bool, error) {
 func (db *Connector) explainAbleSQL(sql string) (string, error) {
 	stmt, err := sqlparser.Parse(sql)
 	if err != nil {
+		// TODO: charset, collation
+		tiStmt, tiErr := ast.TiParse(sql, "", "")
+		if tiErr != nil {
+			common.Log.Error("explainAbleSQL ast.TiParse Error: %v", tiErr)
+			return "", tiErr
+		}
+
+		var isSelect bool
+		for _, st := range tiStmt {
+			switch st.(type) {
+			case *tidb.SelectStmt, *tidb.UnionStmt:
+				isSelect = true
+			default:
+				isSelect = false
+			}
+			if !isSelect {
+				break
+			}
+		}
+
+		if isSelect {
+			return sql, nil
+		}
+
 		common.Log.Error("explainAbleSQL sqlparser.Parse Error: %v", err)
-		return sql, err
+		return "", err
 	}
 
 	switch stmt.(type) {
diff --git a/database/explain_test.go b/database/explain_test.go
index 6cd6e60..1aa5715 100644
--- a/database/explain_test.go
+++ b/database/explain_test.go
@@ -83,6 +83,7 @@ var sqls = []string{
 	`select first_name,last_name,email from customer natural right join address;`,
 	`select first_name,last_name,email from customer STRAIGHT_JOIN address on customer.address_id=address.address_id;`,
 	`select ID,name from (select address from customer_list where SID=1 order by phone limit 50,10) a join customer_list l on (a.address=l.address) join city c on (c.city=l.city) order by phone desc;`,
+	`SELECT a.table_name 表名, a.table_comment 表说明, b.COLUMN_NAME 字段名, b.column_comment 字段说明, b.column_type 字段类型, b.column_key 约束 FROM information_schema.TABLES a LEFT JOIN information_schema. COLUMNS b ON a.table_name = b.TABLE_NAME WHERE a.table_schema IN ('a') AND b.column_comment LIKE '%一%' ORDER BY a.table_name`,
 }
 
 var exp = []string{
@@ -2452,3 +2453,11 @@ func TestSupportExplainWrite(t *testing.T) {
 		t.Error(err)
 	}
 }
+
+func TestExplainAbleSQL(t *testing.T) {
+	for _, sql := range sqls {
+		if _, err := connTest.explainAbleSQL(sql); err != nil {
+			t.Errorf("SQL: %s, not explain able", sql)
+		}
+	}
+}
diff --git a/genver.sh b/genver.sh
index 04f608c..5255025 100755
--- a/genver.sh
+++ b/genver.sh
@@ -1,20 +1,5 @@
 #!/bin/bash
 
-## Go version check
-GO_VERSION_MIN=$1
-echo "==> Checking that build is using go version >= ${GO_VERSION_MIN}..."
-
-GO_VERSION=$(go version | grep -o 'go[0-9]\+\.[0-9]\+\(\.[0-9]\+\)\?' | tr -d 'go')
-
-IFS="." read -r -a GO_VERSION_ARR <<<"$GO_VERSION"
-IFS="." read -r -a GO_VERSION_REQ <<<"$GO_VERSION_MIN"
-
-if [[ ${GO_VERSION_ARR[0]} -lt ${GO_VERSION_REQ[0]} || (${GO_VERSION_ARR[0]} -eq ${GO_VERSION_REQ[0]} && (${GO_VERSION_ARR[1]} -lt ${GO_VERSION_REQ[1]} || (${GO_VERSION_ARR[1]} -eq ${GO_VERSION_REQ[1]} && ${GO_VERSION_ARR[2]} -lt ${GO_VERSION_REQ[2]}))) ]] \
-    ; then
-    echo "requires go $GO_VERSION_MIN to build; found $GO_VERSION."
-    exit 1
-fi
-
 ## Generate Repository Version
 version="$(git log --date=iso --pretty=format:"%cd" -1) $(git describe --tags --always)"
 if [ "X${version}" == "X" ]; then
-- 
GitLab