From 61325627faf869c501c28122517617bec1706752 Mon Sep 17 00:00:00 2001 From: Chen Mulong Date: Mon, 29 Jul 2019 10:18:26 +0800 Subject: [PATCH] Fix quirk of isolation2 test (#8245) * Fix quirk of isolation2 test - Fix the line continuation behaviour and document Instead of matching "; *$" to detect the end of a sql end, the original code was using ";$". This creates a totally different output for tests like: a) SELECT * FROM t1; SELECT * FROM t2; b) SELECT * FROM t1; SELECT * FROM t2; For a), two SELECT statements will be sent separately. For b), they will be sent together as "SELECT * FROM t1; SELECT * FROM t2;" Spaces are playing too much magic here, some of them could be a bug in the test itself. For example, the oom_query test should NOT pass since it misses error output in the expected. This change unifies the behaviours by matching with "; *$". So if a line is end with ';', no matter spaces follows it or not, the statement will be sent at once. - Document the known issue of multi SQL in one line. This issue was found during debugging pipeline failure of #8211 Authored-by: Chen Mulong --- .../expected/dml_on_root_locks_all_parts.out | 4 +++- src/test/isolation2/expected/oom_query.out | 4 +++- ...reateidx_while_reindex_idx_heap_bitmap.out | 6 +++-- ...createidx_while_reindex_idx_heap_btree.out | 6 +++-- ...dextable_while_reindex_idx_heap_bitmap.out | 6 +++-- .../vacuum_while_reindex_heap_btree.out | 6 +++-- .../vacuum_while_reindex_heap_btree_toast.out | 4 +++- src/test/isolation2/sql/oom_query.sql | 3 +-- src/test/isolation2/sql_isolation_testcase.py | 23 ++++++++++++++++++- 9 files changed, 48 insertions(+), 14 deletions(-) diff --git a/src/test/isolation2/expected/dml_on_root_locks_all_parts.out b/src/test/isolation2/expected/dml_on_root_locks_all_parts.out index e93a6091c8..91e121e219 100644 --- a/src/test/isolation2/expected/dml_on_root_locks_all_parts.out +++ b/src/test/isolation2/expected/dml_on_root_locks_all_parts.out @@ -49,7 +49,9 @@ DELETE 0 1: BEGIN; BEGIN -- UPDATE will acquire Exclusive lock on root and leaf partition on QD. -1: UPDATE part_tbl SET c = 1; SELECT GRANTED FROM pg_locks WHERE relation = 'part_tbl_1_prt_1'::regclass::oid AND mode='ExclusiveLock' AND gp_segment_id=-1 AND locktype='relation'; +1: UPDATE part_tbl SET c = 1; +UPDATE 0 +SELECT GRANTED FROM pg_locks WHERE relation = 'part_tbl_1_prt_1'::regclass::oid AND mode='ExclusiveLock' AND gp_segment_id=-1 AND locktype='relation'; granted --------- t diff --git a/src/test/isolation2/expected/oom_query.out b/src/test/isolation2/expected/oom_query.out index aafe04c3e6..35c373c5df 100644 --- a/src/test/isolation2/expected/oom_query.out +++ b/src/test/isolation2/expected/oom_query.out @@ -1 +1,3 @@ -SELECT Count(*) FROM (SELECT l0.l_partkey FROM (lineitem l0 LEFT OUTER JOIN lineitem l1 ON l0.l_partkey = l1.l_partkey LEFT OUTER JOIN lineitem l2 ON l1.l_partkey = l2.l_partkey LEFT OUTER JOIN lineitem l3 ON l2.l_partkey = l3.l_partkey LEFT OUTER JOIN lineitem l4 ON l3.l_partkey = l4.l_partkey LEFT OUTER JOIN lineitem l5 ON l4.l_partkey = l5.l_partkey LEFT OUTER JOIN lineitem l6 ON l5.l_partkey = l6.l_partkey LEFT OUTER JOIN lineitem l7 ON l6.l_partkey = l7.l_partkey LEFT OUTER JOIN lineitem l8 ON l7.l_partkey = l8.l_partkey LEFT OUTER JOIN lineitem l9 ON l8.l_partkey = l9.l_partkey LEFT OUTER JOIN lineitem l10 ON l9.l_partkey = l10.l_partkey LEFT OUTER JOIN lineitem l11 ON l10.l_partkey = l11.l_partkey LEFT OUTER JOIN lineitem l12 ON l11.l_partkey = l12.l_partkey LEFT OUTER JOIN lineitem l13 ON l12.l_partkey = l13.l_partkey LEFT OUTER JOIN lineitem l14 ON l13.l_partkey = l14.l_partkey LEFT OUTER JOIN lineitem l15 ON l14.l_partkey = l15.l_partkey) ORDER BY l0.l_partkey) AS foo; +SELECT Count(*) FROM (SELECT l0.l_partkey FROM (lineitem l0 LEFT OUTER JOIN lineitem l1 ON l0.l_partkey = l1.l_partkey LEFT OUTER JOIN lineitem l2 ON l1.l_partkey = l2.l_partkey LEFT OUTER JOIN lineitem l3 ON l2.l_partkey = l3.l_partkey LEFT OUTER JOIN lineitem l4 ON l3.l_partkey = l4.l_partkey LEFT OUTER JOIN lineitem l5 ON l4.l_partkey = l5.l_partkey LEFT OUTER JOIN lineitem l6 ON l5.l_partkey = l6.l_partkey LEFT OUTER JOIN lineitem l7 ON l6.l_partkey = l7.l_partkey LEFT OUTER JOIN lineitem l8 ON l7.l_partkey = l8.l_partkey LEFT OUTER JOIN lineitem l9 ON l8.l_partkey = l9.l_partkey LEFT OUTER JOIN lineitem l10 ON l9.l_partkey = l10.l_partkey LEFT OUTER JOIN lineitem l11 ON l10.l_partkey = l11.l_partkey LEFT OUTER JOIN lineitem l12 ON l11.l_partkey = l12.l_partkey LEFT OUTER JOIN lineitem l13 ON l12.l_partkey = l13.l_partkey LEFT OUTER JOIN lineitem l14 ON l13.l_partkey = l14.l_partkey LEFT OUTER JOIN lineitem l15 ON l14.l_partkey = l15.l_partkey) ORDER BY l0.l_partkey) AS foo; +ERROR: Out of memory (seg0 slice17 127.0.0.1:25432 pid=29534) +DETAIL: Per-query VM protect limit reached: current limit is 2048 kB, requested 1048584 bytes, available 0 MB diff --git a/src/test/isolation2/expected/reindex/createidx_while_reindex_idx_heap_bitmap.out b/src/test/isolation2/expected/reindex/createidx_while_reindex_idx_heap_bitmap.out index 21bb5629f5..15bbec1e6a 100644 --- a/src/test/isolation2/expected/reindex/createidx_while_reindex_idx_heap_bitmap.out +++ b/src/test/isolation2/expected/reindex/createidx_while_reindex_idx_heap_bitmap.out @@ -1,8 +1,10 @@ DROP TABLE IF EXISTS reindex_crtab_heap_bitmap; DROP -CREATE TABLE reindex_crtab_heap_bitmap (a INT); insert into reindex_crtab_heap_bitmap select generate_series(1,1000); -CREATE 1000 +CREATE TABLE reindex_crtab_heap_bitmap (a INT); +CREATE +insert into reindex_crtab_heap_bitmap select generate_series(1,1000); +INSERT 1000 insert into reindex_crtab_heap_bitmap select generate_series(1,1000); INSERT 1000 create index idx_reindex_crtab_heap_bitmap on reindex_crtab_heap_bitmap(a); diff --git a/src/test/isolation2/expected/reindex/createidx_while_reindex_idx_heap_btree.out b/src/test/isolation2/expected/reindex/createidx_while_reindex_idx_heap_btree.out index d080c743b2..7bae7e1d94 100644 --- a/src/test/isolation2/expected/reindex/createidx_while_reindex_idx_heap_btree.out +++ b/src/test/isolation2/expected/reindex/createidx_while_reindex_idx_heap_btree.out @@ -1,8 +1,10 @@ DROP TABLE IF EXISTS reindex_crtab_heap_btree; DROP -CREATE TABLE reindex_crtab_heap_btree (a INT); insert into reindex_crtab_heap_btree select generate_series(1,1000); -CREATE 1000 +CREATE TABLE reindex_crtab_heap_btree (a INT); +CREATE +insert into reindex_crtab_heap_btree select generate_series(1,1000); +INSERT 1000 insert into reindex_crtab_heap_btree select generate_series(1,1000); INSERT 1000 create index idx_reindex_crtab_heap_btree on reindex_crtab_heap_btree(a); diff --git a/src/test/isolation2/expected/reindex/reindextable_while_reindex_idx_heap_bitmap.out b/src/test/isolation2/expected/reindex/reindextable_while_reindex_idx_heap_bitmap.out index bd1c0d158f..0a9689cb03 100644 --- a/src/test/isolation2/expected/reindex/reindextable_while_reindex_idx_heap_bitmap.out +++ b/src/test/isolation2/expected/reindex/reindextable_while_reindex_idx_heap_bitmap.out @@ -1,8 +1,10 @@ DROP TABLE IF EXISTS reindex_crtab_heap_bitmap; DROP -CREATE TABLE reindex_crtab_heap_bitmap (a INT); insert into reindex_crtab_heap_bitmap select generate_series(1,1000); -CREATE 1000 +CREATE TABLE reindex_crtab_heap_bitmap (a INT); +CREATE +insert into reindex_crtab_heap_bitmap select generate_series(1,1000); +INSERT 1000 insert into reindex_crtab_heap_bitmap select generate_series(1,1000); INSERT 1000 create index idx_reindex_crtab_heap_bitmap on reindex_crtab_heap_bitmap USING BITMAP(a); diff --git a/src/test/isolation2/expected/reindex/vacuum_while_reindex_heap_btree.out b/src/test/isolation2/expected/reindex/vacuum_while_reindex_heap_btree.out index f3502cfeab..c049f48081 100644 --- a/src/test/isolation2/expected/reindex/vacuum_while_reindex_heap_btree.out +++ b/src/test/isolation2/expected/reindex/vacuum_while_reindex_heap_btree.out @@ -1,8 +1,10 @@ DROP TABLE IF EXISTS reindex_heap; DROP -CREATE TABLE reindex_heap (a INT); insert into reindex_heap select generate_series(1,1000); -CREATE 1000 +CREATE TABLE reindex_heap (a INT); +CREATE +insert into reindex_heap select generate_series(1,1000); +INSERT 1000 insert into reindex_heap select generate_series(1,1000); INSERT 1000 create index idx_btree_reindex_heap on reindex_heap(a); diff --git a/src/test/isolation2/expected/reindex/vacuum_while_reindex_heap_btree_toast.out b/src/test/isolation2/expected/reindex/vacuum_while_reindex_heap_btree_toast.out index 4aa7219c51..6b6254e723 100644 --- a/src/test/isolation2/expected/reindex/vacuum_while_reindex_heap_btree_toast.out +++ b/src/test/isolation2/expected/reindex/vacuum_while_reindex_heap_btree_toast.out @@ -1,8 +1,10 @@ DROP TABLE IF EXISTS reindex_toast_heap; DROP -CREATE TABLE reindex_toast_heap (a text, b int); alter table reindex_toast_heap alter column a set storage external; +CREATE TABLE reindex_toast_heap (a text, b int); CREATE +alter table reindex_toast_heap alter column a set storage external; +ALTER insert into reindex_toast_heap select repeat('123456789',10000), i from generate_series(1,100) i; INSERT 100 create index idx_btree_reindex_toast_heap on reindex_toast_heap(b); diff --git a/src/test/isolation2/sql/oom_query.sql b/src/test/isolation2/sql/oom_query.sql index 65dd1999a4..8b04ccedb3 100644 --- a/src/test/isolation2/sql/oom_query.sql +++ b/src/test/isolation2/sql/oom_query.sql @@ -31,5 +31,4 @@ SELECT Count(*) ON l13.l_partkey = l14.l_partkey LEFT OUTER JOIN lineitem l15 ON l14.l_partkey = l15.l_partkey) - ORDER BY l0.l_partkey) AS foo; - + ORDER BY l0.l_partkey) AS foo; diff --git a/src/test/isolation2/sql_isolation_testcase.py b/src/test/isolation2/sql_isolation_testcase.py index beca1b5d63..ce36d90da0 100644 --- a/src/test/isolation2/sql_isolation_testcase.py +++ b/src/test/isolation2/sql_isolation_testcase.py @@ -471,7 +471,7 @@ class SQLIsolationExecutor(object): command_part = line if command_part == "" or command_part == "\n": print >>output_file - elif command_part.endswith(";\n") or re.match(r"^\d+[q\\<]:$", line) or re.match(r"^-?\d+[SU][q\\<]:$", line): + elif re.match(r".*;\s*$", command_part) or re.match(r"^\d+[q\\<]:$", line) or re.match(r"^-?\d+[SU][q\\<]:$", line): command += command_part try: self.process_command(command, output_file) @@ -616,6 +616,27 @@ class SQLIsolationTestCase: -- example contents for file.sql: create function some_test_function() returning void ... include: path/to/some/file.sql; select some_helper_function(); + + Line continuation: + If a line is not ended by a semicolon ';' which is followed by 0 or more spaces, the line will be combined with next line and + sent together as a single statement. + + e.g.: Send to the server separately: + 1: SELECT * FROM t1; -> send "SELECT * FROM t1;" + SELECT * FROM t2; -> send "SELECT * FROM t2;" + + e.g.: Send to the server once: + 1: SELECT * FROM + t1; SELECT * FROM t2; -> "send SELECT * FROM t1; SELECT * FROM t2;" + + ATTENTION: + Send multi SQL statements once: + Multi SQL statements can be sent at once, but there are some known issues. Generally only the last query result will be printed. + But due to the difficulties of dealing with semicolons insides quotes, we always echo the first SQL command instead of the last + one if query() returns None. This created some strange issues like: + + CREATE TABLE t1 (a INT); INSERT INTO t1 SELECT generate_series(1,1000); + CREATE 1000 (Should be INSERT 1000, but here the CREATE is taken due to the limitation) """ def run_sql_file(self, sql_file, out_file = None, out_dir = None, optimizer = None): -- GitLab