From f8734d9d62b4eb14e61c0e5d64d8e67c0184ac67 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 16 Feb 2017 16:58:45 +0200 Subject: [PATCH] Move tests on 'demoprot' from TINC to contrib, where the source is. I had to change some of the tests that used 4 tests rows, to have more (9), to make them pass on a 3 segment cluster. If you only insert 4 rows to an external table, some segments might not receive any rows, and reading from the same file later will fail because the file wasn't created. I also changed the queries that tested that rows are distributed evenly, to work with a different number of segments. Hook up the new tests to installcheck-world. --- GNUmakefile.in | 1 + contrib/extprotocol/Makefile | 1 + .../extprotocol/expected/exttableext.out | 804 ++++++------------ contrib/extprotocol/expected/setup.out | 71 ++ .../extprotocol}/sql/exttableext.sql | 117 ++- .../extprotocol/sql/setup.sql | 46 - .../basic/exttab/exttableext/__init__.py | 0 .../exttableext/expected/setup/query00.ans | 160 ---- .../basic/exttab/exttableext/init_file | 5 - .../sql/setup/removeOutputFile.sql | 9 - .../basic/exttab/exttableext/test_exttable.py | 45 - 11 files changed, 398 insertions(+), 861 deletions(-) rename src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/expected/exttableext.ans => contrib/extprotocol/expected/exttableext.out (82%) create mode 100644 contrib/extprotocol/expected/setup.out rename {src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext => contrib/extprotocol}/sql/exttableext.sql (94%) rename src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/setup/query00.sql => contrib/extprotocol/sql/setup.sql (51%) delete mode 100644 src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/__init__.py delete mode 100644 src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/expected/setup/query00.ans delete mode 100644 src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/init_file delete mode 100644 src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/setup/removeOutputFile.sql delete mode 100755 src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/test_exttable.py diff --git a/GNUmakefile.in b/GNUmakefile.in index 6dd744caa5..396eedbdb2 100644 --- a/GNUmakefile.in +++ b/GNUmakefile.in @@ -121,6 +121,7 @@ installcheck-world: $(MAKE) -C src/pl installcheck #$(MAKE) -C src/interfaces/ecpg installcheck #$(MAKE) -C contrib installcheck + $(MAKE) -C contrib/extprotocol installcheck $(MAKE) -C gpAux/extensions installcheck $(MAKE) -C src/bin/gpfdist installcheck diff --git a/contrib/extprotocol/Makefile b/contrib/extprotocol/Makefile index 8ac73895a4..bb45970e80 100644 --- a/contrib/extprotocol/Makefile +++ b/contrib/extprotocol/Makefile @@ -1,5 +1,6 @@ MODULE_big = gpextprotocol OBJS = gpextprotocol.o +REGRESS = setup exttableext PG_CPPFLAGS = -I$(libpq_srcdir) PG_LIBS = $(libpq_pgport) diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/expected/exttableext.ans b/contrib/extprotocol/expected/exttableext.out similarity index 82% rename from src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/expected/exttableext.ans rename to contrib/extprotocol/expected/exttableext.out index 9affb55d35..9008ca9156 100644 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/expected/exttableext.ans +++ b/contrib/extprotocol/expected/exttableext.out @@ -1,22 +1,17 @@ SET search_path TO 'exttableext'; -SET -- Test 3: create RET and WET using created protocol -- Create external RET and WET DROP EXTERNAL TABLE IF EXISTS exttabtest_w; NOTICE: table "exttabtest_w" does not exist, skipping -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w(like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_r; NOTICE: table "exttabtest_r" does not exist, skipping -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE exttabtest_r(like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- Checking pg_exttable for new created RET and WET select location,fmttype,fmtopts,encoding,writable from pg_exttable where reloid='exttabtest_r'::regclass @@ -34,7 +29,6 @@ CREATE EXTERNAL TABLE (0 rows) INSERT INTO exttabtest_w (SELECT * FROM exttabtest); -INSERT 0 100 -- read from RET SELECT * FROM exttabtest_r EXCEPT ALL @@ -43,54 +37,46 @@ INSERT 0 100 ----+------+--------+-------- (0 rows) - -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from exttabtest_r - GROUP BY 1 ORDER BY 1; - gp_segment_id | count ----------------+------- - 0 | 50 - 1 | 50 -(2 rows) +-- verify data should be evenly distributed +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; + ?column? +---------- + f +(1 row) -- Test 4.1: create uni-directional write protocol -- create WET using created protocol DROP PROTOCOL IF EXISTS demoprot CASCADE; NOTICE: drop cascades to external table exttabtest_r NOTICE: drop cascades to external table exttabtest_w -DROP PROTOCOL CREATE PROTOCOL demoprot ( writefunc = write_to_file_stable ); -CREATE PROTOCOL -- Create WET with uni-directional protocol DROP EXTERNAL TABLE IF EXISTS exttabtest_w_uni; NOTICE: table "exttabtest_w_uni" does not exist, skipping -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w_uni(like exttabtest) LOCATION('demoprot://exttabtest_uni.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE -- write to WET INSERT INTO exttabtest_w_uni (SELECT * FROM exttabtest); -INSERT 0 100 -- Test 4.2: create uni-directional read protocol -- create RET using created protocol DROP PROTOCOL IF EXISTS demoprot CASCADE; NOTICE: drop cascades to external table exttabtest_w_uni -DROP PROTOCOL CREATE PROTOCOL demoprot ( readfunc = read_from_file_stable ); -CREATE PROTOCOL -- Create RET with uni-directional protocol DROP EXTERNAL TABLE IF EXISTS exttabtest_r_uni; NOTICE: table "exttabtest_r_uni" does not exist, skipping -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE exttabtest_r_uni(like exttabtest) LOCATION('demoprot://exttabtest_uni.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- read from RET SELECT * FROM exttabtest_r_uni EXCEPT ALL @@ -109,30 +95,23 @@ CREATE EXTERNAL TABLE -- Otherwise "ERROR: demoprot_import: could not open file " will be thrown. DROP PROTOCOL IF EXISTS demoprot CASCADE; NOTICE: drop cascades to external table exttabtest_r_uni -DROP PROTOCOL CREATE PROTOCOL demoprot ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -CREATE PROTOCOL DROP EXTERNAL TABLE IF EXISTS exttabtest_w_dist; NOTICE: table "exttabtest_w_dist" does not exist, skipping -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w_dist(like exttabtest) LOCATION('demoprot://exttabtest_dist.txt') FORMAT 'text' DISTRIBUTED BY (value2); -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_r_dist; NOTICE: table "exttabtest_r_dist" does not exist, skipping -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE exttabtest_r_dist(like exttabtest) LOCATION('demoprot://exttabtest_dist.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- write to WET INSERT INTO exttabtest_w_dist (SELECT * FROM exttabtest); -INSERT 0 100 -- read from RET SELECT * FROM exttabtest_r_dist EXCEPT ALL @@ -141,36 +120,32 @@ INSERT 0 100 ----+------+--------+-------- (0 rows) - -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from exttabtest_r_dist - GROUP BY 1 ORDER BY 1; - gp_segment_id | count ----------------+------- - 0 | 51 - 1 | 49 -(2 rows) +-- verify data should be evenly distributed +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_dist group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; + ?column? +---------- + f +(1 row) -- Test 6: using two urls and using CSV format DROP EXTERNAL TABLE IF EXISTS exttabtest_r_2url; NOTICE: table "exttabtest_r_2url" does not exist, skipping -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE exttabtest_r_2url (like exttabtest) LOCATION('demoprot://exttabtest_2url_1.csv', 'demoprot://exttabtest_2url_2.csv') FORMAT 'csv'; -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_w_2url; NOTICE: table "exttabtest_w_2url" does not exist, skipping -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w_2url (like exttabtest) LOCATION('demoprot://exttabtest_2url_1.csv', 'demoprot://exttabtest_2url_2.csv') FORMAT 'csv' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE -- write to WET INSERT INTO exttabtest_w_2url (SELECT * FROM exttabtest); -INSERT 0 100 -- read from RET SELECT * FROM exttabtest_r_2url EXCEPT ALL @@ -180,36 +155,32 @@ INSERT 0 100 (0 rows) -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from exttabtest_r_2url - GROUP BY 1 ORDER BY 1; - gp_segment_id | count ----------------+------- - 0 | 50 - 1 | 50 -(2 rows) +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_2url group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; + ?column? +---------- + f +(1 row) -- Check the output file at each segments -- ! gpssh -f allsegs ls -l /data/hhuang/MAIN/main_debug/primary/gpseg*/exttabtest_2url*.csv -- Test 7: using two urls and text format DROP EXTERNAL TABLE IF EXISTS exttabtest_r_2url; -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE exttabtest_r_2url (like exttabtest) LOCATION('demoprot://exttabtest_2url_1.txt', 'demoprot://exttabtest_2url_2.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_w_2url; -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w_2url (like exttabtest) LOCATION('demoprot://exttabtest_2url_1.txt', 'demoprot://exttabtest_2url_2.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE -- write to WET INSERT INTO exttabtest_w_2url (SELECT * FROM exttabtest); -INSERT 0 100 -- read from RET SELECT * FROM exttabtest_r_2url EXCEPT ALL @@ -219,20 +190,20 @@ INSERT 0 100 (0 rows) -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from exttabtest_r_2url - GROUP BY 1 ORDER BY 1; - gp_segment_id | count ----------------+------- - 0 | 50 - 1 | 50 -(2 rows) +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_2url group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; + ?column? +---------- + f +(1 row) -- Checking the output files on segments -- ! gpssh -f allsegs ls -l /data/hhuang/MAIN/main_debug/primary/gpseg*/exttabtest_2url*.txt -- Test 8: Negative - using 5 urls, exceeding number of primary segments (4) DROP EXTERNAL TABLE IF EXISTS exttabtest_w_5url; NOTICE: table "exttabtest_w_5url" does not exist, skipping -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w_5url (like exttabtest) LOCATION('demoprot://exttabtest_5url_1.txt', 'demoprot://exttabtest_5url_2.txt', @@ -241,10 +212,10 @@ DROP EXTERNAL TABLE 'demoprot://exttabtest_5url_5.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE -- write to WET INSERT INTO exttabtest_w_5url (SELECT * FROM exttabtest); -psql:/path/sql_file:1: ERROR: External table has more URLs than available primary segments that can write into them (seg0 rh55-qavm55:7532 pid=1235) +ERROR: External table has more URLs than available primary segments that can write into them (seg0 rh55-qavm55:7532 pid=1235) +-- @skip Skipping this test because of a duplicate key violation error. -- Test 9: Negative - duplicte protocol name (check pg_extprotocol) SELECT count(*) FROM pg_extprotocol WHERE ptcname='demoprot'; count @@ -256,13 +227,13 @@ psql:/path/sql_file:1: ERROR: External table has more URLs than available prima readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -psql:/path/sql_file:1: ERROR: protocol "demoprot" already exists +ERROR: protocol "demoprot" already exists -- Test 10: Negative: create external table using non-existing protocol CREATE WRITABLE EXTERNAL TABLE exttabtest_w_neg10 (like exttabtest) LOCATION('demoprot_nonexist://exttabtest_neg10.txt') FORMAT 'text' DISTRIBUTED BY (id); -psql:/path/sql_file:1: ERROR: protocol "demoprot_nonexist" does not exist +ERROR: protocol "demoprot_nonexist" does not exist -- Test 11: Negative - Using invalid protocol attribute name -- attribute names must be readproc, write proc, and validatorproc DROP PROTOCOL IF EXISTS demoprot CASCADE; @@ -271,60 +242,53 @@ NOTICE: drop cascades to external table exttabtest_w_2url NOTICE: drop cascades to external table exttabtest_r_2url NOTICE: drop cascades to external table exttabtest_r_dist NOTICE: drop cascades to external table exttabtest_w_dist -DROP PROTOCOL CREATE PROTOCOL demoprot ( readfunction = read_from_file, writefunction = write_to_file ); -psql:/path/sql_file:1: ERROR: protocol attribute "readfunction" not recognized +ERROR: protocol attribute "readfunction" not recognized -- Test 12: Negatvie - using undefined function when defining protocol DROP PROTOCOL IF EXISTS demoprot CASCADE; -psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping -DROP PROTOCOL +NOTICE: protocol "demoprot" does not exist, skipping CREATE PROTOCOL demoprot ( readfunc = read_from_file_badname, writefunc = write_to_file_badname ); -psql:/path/sql_file:1: ERROR: function read_from_file_badname() does not exist +ERROR: function read_from_file_badname() does not exist -- Test 13: Negatvie - syntax error: missing '=' when defining protocol DROP PROTOCOL IF EXISTS demoprot CASCADE; -psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping -DROP PROTOCOL +NOTICE: protocol "demoprot" does not exist, skipping CREATE PROTOCOL demoprot ( readfunc read_from_file, writefunc write_to_file ); -psql:/path/sql_file:1: ERROR: syntax error at or near "read_from_file" +ERROR: syntax error at or near "read_from_file" LINE 2: readfunc read_from_file, ^ +-- @skip Test disabled in cdbfast as well -- Test 14: Negative - switching read function and write function -- This is user error. GPDB should display meaningful error message. +-- Not running this test. Comment from cdbfast - Comment this out since it doesn't make much sense and it creates big output files DROP PROTOCOL IF EXISTS demoprot CASCADE; -psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping -DROP PROTOCOL +NOTICE: protocol "demoprot" does not exist, skipping CREATE PROTOCOL demoprot ( writefunc = read_from_file_stable, readfunc = write_to_file_stable ); -CREATE PROTOCOL DROP EXTERNAL TABLE IF EXISTS exttabtest_w_switched; NOTICE: table "exttabtest_r_switched" does not exist, skipping -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w_switched (like exttabtest) LOCATION('demoprot://exttabtest_switched.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_r_switched; NOTICE: table "exttabtest_r_switched" does not exist, skipping -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE exttabtest_r_switched(like exttabtest) LOCATION('demoprot://exttabtest_switched.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- write to WET INSERT INTO exttabtest_w_switched (SELECT * FROM exttabtest); -psql:/path/sql_file:1: ERROR: demoprot_import: could not open file "exttabtest_switched.txt" for reading: No such file or directory (seg0 rh55-qavm55:7532 pid=32378) +ERROR: demoprot_import: could not open file "exttabtest_switched.txt" for reading: No such file or directory (seg0 rh55-qavm55:7532 pid=32378) -- read from RET --SELECT * FROM exttabtest_r_switched --EXCEPT ALL @@ -332,160 +296,149 @@ psql:/path/sql_file:1: ERROR: demoprot_import: could not open file "exttabtest_ -- Test 15: Negative - circular reference -- write to WET while selecting from RET, and WET and RET are using the same data source files DROP PROTOCOL IF EXISTS demoprot CASCADE; -psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping +NOTICE: protocol "demoprot" does not exist, skipping +NOTICE: drop cascades to external table exttabtest_r_switched NOTICE: drop cascades to external table exttabtest_w_switched -DROP PROTOCOL CREATE PROTOCOL demoprot ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -CREATE PROTOCOL DROP EXTERNAL TABLE IF EXISTS exttabtest_w_circle; NOTICE: table "exttabtest_w_circle" does not exist, skipping -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w_circle(like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_r_circle; NOTICE: table "exttabtest_r_circle" does not exist, skipping -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE exttabtest_r_circle(like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- First to create exttabtest.txt using WET INSERT INTO exttabtest_w_circle (SELECT * FROM exttabtest); -INSERT 0 100 -- write to WET while reading from RET, -- using limit to avoid infinit loop -- This is to test that using RET and WET inappropriately can get yourself into trouble INSERT INTO exttabtest_w_circle (SELECT * FROM exttabtest_r_circle order by id limit 100); -INSERT 0 100 -- Test 16: Negative - invalid URL: missing path drop external table if exists exttabtest_w_misspath; NOTICE: table "exttabtest_w_misspath" does not exist, skipping -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w_misspath(like exttabtest) LOCATION('demoprot://') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE insert into exttabtest_w_misspath (select * from exttabletest); -psql:/path/sql_file:1: ERROR: relation "exttabletest" does not exist +ERROR: relation "exttabletest" does not exist LINE 1: insert into exttabtest_w_misspath (select * from exttabletes... ^ -- Test 17: Negative - invalid URL: missing protocol CREATE READABLE EXTERNAL TABLE exttabtest_r_missprot(like exttabtest) LOCATION('exttabtest.txt') FORMAT 'text'; -psql:/path/sql_file:1: ERROR: invalid URI 'exttabtest.txt' : undefined structure +ERROR: invalid URI 'exttabtest.txt' : undefined structure -- Test 18: Negative - invalid URL: invalid path CREATE READABLE EXTERNAL TABLE exttabtest_r_invalidpath(like exttabtest) LOCATION('demoprot:\\exttabtest.txt') FORMAT 'text'; -psql:/path/sql_file:1: ERROR: invalid URI 'demoprot:\\exttabtest.txt' : undefined structure +ERROR: invalid URI 'demoprot:\\exttabtest.txt' : undefined structure -- Test 19: Negative - invalid URL: invalid protocol name CREATE READABLE EXTERNAL TABLE exttabtest_r_invalidprot(like exttabtest) LOCATION('badprotocol://exttabtest.txt') FORMAT 'text'; -psql:/path/sql_file:1: ERROR: protocol "badprotocol" does not exist --- Test 20: Small dataset - 4 records - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_4records; -NOTICE: table "exttabtest_w_4records" does not exist, skipping -DROP EXTERNAL TABLE - CREATE WRITABLE EXTERNAL TABLE exttabtest_w_4records (like exttabtest) - LOCATION('demoprot://exttabtest_4records.txt') +ERROR: protocol "badprotocol" does not exist +-- Test 20: Small dataset - 20 records + DROP EXTERNAL TABLE IF EXISTS exttabtest_w_20records; +NOTICE: table "exttabtest_w_20records" does not exist, skipping + CREATE WRITABLE EXTERNAL TABLE exttabtest_w_20records (like exttabtest) + LOCATION('demoprot://exttabtest_20records.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_4records; -NOTICE: table "exttabtest_r_4records" does not exist, skipping -DROP EXTERNAL TABLE - CREATE READABLE EXTERNAL TABLE exttabtest_r_4records (like exttabtest) - LOCATION('demoprot://exttabtest_4records.txt') + DROP EXTERNAL TABLE IF EXISTS exttabtest_r_20records; +NOTICE: table "exttabtest_r_20records" does not exist, skipping + CREATE READABLE EXTERNAL TABLE exttabtest_r_20records (like exttabtest) + LOCATION('demoprot://exttabtest_20records.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- write to WET - INSERT INTO exttabtest_w_4records (SELECT * FROM exttabtest where id<=4); -INSERT 0 4 + INSERT INTO exttabtest_w_20records (SELECT * FROM exttabtest where id<=20); -- read from RET - SELECT * FROM exttabtest_r_4records order by id; + SELECT * FROM exttabtest_r_20records order by id; id | name | value1 | value2 -----+-------+--------+-------- +----+--------+--------+-------- 1 | name1 | 2 | 3 2 | name2 | 4 | 6 3 | name3 | 6 | 9 4 | name4 | 8 | 12 -(4 rows) + 5 | name5 | 10 | 15 + 6 | name6 | 12 | 18 + 7 | name7 | 14 | 21 + 8 | name8 | 16 | 24 + 9 | name9 | 18 | 27 + 10 | name10 | 20 | 30 + 11 | name11 | 22 | 33 + 12 | name12 | 24 | 36 + 13 | name13 | 26 | 39 + 14 | name14 | 28 | 42 + 15 | name15 | 30 | 45 + 16 | name16 | 32 | 48 + 17 | name17 | 34 | 51 + 18 | name18 | 36 | 54 + 19 | name19 | 38 | 57 + 20 | name20 | 40 | 60 +(20 rows) -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from exttabtest_r_4records - GROUP BY 1 ORDER BY 1; - gp_segment_id | count ----------------+------- - 0 | 2 - 1 | 2 -(2 rows) +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_20records group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; + ?column? +---------- + f +(1 row) -- Drop External Tables - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_4records; -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_4records; -DROP EXTERNAL TABLE + DROP EXTERNAL TABLE IF EXISTS exttabtest_r_20records; + DROP EXTERNAL TABLE IF EXISTS exttabtest_w_20records; -- Test 21: Small dataset - 1 record DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1record; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_1record" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_w_1record" does not exist, skipping CREATE WRITABLE EXTERNAL TABLE exttabtest_w_1record (like exttabtest) LOCATION('demoprot://exttabtest_1record.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1record; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_1record" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_r_1record" does not exist, skipping CREATE READABLE EXTERNAL TABLE exttabtest_r_1record (like exttabtest) LOCATION('demoprot://exttabtest_1record.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- write to WET INSERT INTO exttabtest_w_1record (SELECT * FROM exttabtest where id = 4); -INSERT 0 1 -- read from RET -- The implemented example protocol (demoprot) requires data file must be available -- (even it is empty) for each (primary) segment. -- This is the limitation of this example (MPP-13811) -- and will cause following SElECT query to fail SELECT * FROM exttabtest_r_1record order by id; -psql:/path/sql_file:1: ERROR: demoprot_import: could not open file "exttabtest_1record.txt" for reading: No such file or directory (seg0 slice1 rh55-qavm55:7532 pid=23845) +ERROR: demoprot_import: could not open file "exttabtest_1record.txt" for reading: No such file or directory (seg0 slice1 rh55-qavm55:7532 pid=23845) DETAIL: External table exttabtest_r_1record, file demoprot://exttabtest_1record.txt -- Drop External Tables DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1record; -DROP EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1record; -DROP EXTERNAL TABLE -- Test 22: Using /dev/null -- Using /dev/null for output and input DROP EXTERNAL TABLE IF EXISTS exttabtest_w_null; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_null" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_w_null" does not exist, skipping CREATE WRITABLE EXTERNAL TABLE exttabtest_w_null (like exttabtest) LOCATION('demoprot:///dev/null') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_r_null; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_null" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_r_null" does not exist, skipping CREATE READABLE EXTERNAL TABLE exttabtest_r_null (like exttabtest) LOCATION('demoprot:///dev/null') FORMAT 'text'; -CREATE EXTERNAL TABLE -- write to WET INSERT INTO exttabtest_w_null (SELECT * FROM exttabtest); -INSERT 0 100 -- read from RET SELECT count(id) FROM exttabtest_r_null; count @@ -496,28 +449,21 @@ INSERT 0 100 -- Test 23: Performance - 1M records -- Load 1M rows of data TRUNCATE TABLE exttabtest; -TRUNCATE TABLE INSERT INTO exttabtest SELECT i, 'name'||i, i*2, i*3 FROM generate_series(1,1000000) i; -INSERT 0 1000000 -- Using demoprot, import and export 1M records DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1M; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_1m" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_w_1m" does not exist, skipping CREATE WRITABLE EXTERNAL TABLE exttabtest_w_1M (like exttabtest) LOCATION('demoprot://exttabtest_1M.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1M; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_1m" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_r_1m" does not exist, skipping CREATE READABLE EXTERNAL TABLE exttabtest_r_1M (like exttabtest) LOCATION('demoprot://exttabtest_1M.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- write to WET INSERT INTO exttabtest_w_1M (SELECT * FROM exttabtest); -INSERT 0 1000000 -- Time: 1369.028 ms -- read from RET SELECT count(id) FROM exttabtest_r_1M; @@ -529,32 +475,25 @@ INSERT 0 1000000 -- Time: 869.793 ms -- Compare to using demoprot protocol and output to /dev/null (no disk IO, thanks to Alan) DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1M_null; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_1m_null" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_w_1m_null" does not exist, skipping CREATE WRITABLE EXTERNAL TABLE exttabtest_w_1M_null (like exttabtest) LOCATION('demoprot:///dev/null') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1M_null; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_1m_null" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_r_1m_null" does not exist, skipping CREATE READABLE EXTERNAL TABLE exttabtest_r_1M_null (like exttabtest) LOCATION('demoprot:///dev/null') FORMAT 'text'; -CREATE EXTERNAL TABLE -- write to WET INSERT INTO exttabtest_w_1M_null (SELECT * FROM exttabtest); -INSERT 0 1000000 -- Time: 1368.173 ms (about same as writing to file - CPU intensive) -- Test 30: UDF dependency - drop UDF when it does not have dependent protocol -- Create new UDF that has no dependent CREATE OR REPLACE FUNCTION write_udf_todrop() RETURNS integer AS '$libdir/gpextprotocol.so', 'demoprot_export' LANGUAGE C STABLE; -CREATE FUNCTION CREATE OR REPLACE FUNCTION read_udf_todrop() RETURNS integer AS '$libdir/gpextprotocol.so', 'demoprot_import' LANGUAGE C STABLE; -CREATE FUNCTION -- Check pg_proc catalog table for new created functions SELECT proname, prolang,proisstrict,provolatile,pronargs,prorettype,prosrc,proacl FROM pg_proc WHERE proname like 'write_udf_todrop' @@ -568,9 +507,7 @@ CREATE FUNCTION -- Drop two UDFs DROP FUNCTION write_udf_todrop(); -DROP FUNCTION DROP FUNCTION read_udf_todrop(); -DROP FUNCTION -- Check pg_proc catalog table for after drop the UDFs SELECT proname, prolang,proisstrict,provolatile,pronargs,prorettype,prosrc,proacl FROM pg_proc WHERE proname like 'write_udf_todrop' @@ -627,18 +564,15 @@ NOTICE: drop cascades to external table exttabtest_w_null NOTICE: drop cascades to external table exttabtest_w_misspath NOTICE: drop cascades to external table exttabtest_r_circle NOTICE: drop cascades to external table exttabtest_w_circle -DROP PROTOCOL CREATE PROTOCOL demoprot ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -CREATE PROTOCOL -- CREATE PROTOCOL -- Drop function cascade. -- Showing notice message of cascade drop of dependent protocol demoprot drop function read_from_file_stable() cascade; -psql:/path/sql_file:1: NOTICE: drop cascades to protocol demoprot -DROP FUNCTION +NOTICE: drop cascades to protocol demoprot -- NOTICE: drop cascades to protocol demoprot -- DROP FUNCTION -- Check pg_proc for function read_from_file_stable @@ -671,38 +605,31 @@ DROP FUNCTION -- Recreate function read_from_file_stable CREATE OR REPLACE FUNCTION read_from_file_stable() RETURNS integer AS '$libdir/gpextprotocol.so', 'demoprot_import' LANGUAGE C STABLE; -CREATE FUNCTION -- Restore (recreate) protocol with the same protocol name demoprot DROP PROTOCOL IF EXISTS demoprot CASCADE; -psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping -DROP PROTOCOL +NOTICE: protocol "demoprot" does not exist, skipping CREATE PROTOCOL demoprot ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -CREATE PROTOCOL -- CREATE PROTOCOL CREATE READABLE EXTERNAL TABLE exttabtest_r(like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w(like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE -- truncate table exttabtest and load 100 records TRUNCATE TABLE exttabtest; -TRUNCATE TABLE INSERT INTO exttabtest SELECT i, 'name'||i, i*2, i*3 FROM generate_series(1,100) i; -INSERT 0 100 -- Check existing WET that using protocol demoprot SELECT * FROM clean_exttabtest_files; stdout -------- (0 rows) + insert into exttabtest_w (select * from exttabtest); -INSERT 0 100 -- Check existing RET that using protocol demoprot select * from exttabtest_r where id <=4 order by id; id | name | value1 | value2 @@ -716,14 +643,10 @@ INSERT 0 100 -- returns 4 records -- Change ext table name when using custom protocol ALTER TABLE exttabtest_r rename to exttabtest_r_newname; -ALTER TABLE -- Check for Add|Drop|Rename column ALTER EXTERNAL TABLE exttabtest_r_newname ADD COLUMN value3 int; -ALTER EXTERNAL TABLE ALTER TABLE exttabtest_r_newname RENAME COLUMN value3 to value3_newname; -ALTER TABLE ALTER EXTERNAL TABLE exttabtest_r_newname DROP COLUMN value3_newname; -ALTER EXTERNAL TABLE -- Check external table is still accessible after alter operation select * from exttabtest_r_newname where id <=4 order by id; id | name | value1 | value2 @@ -761,17 +684,14 @@ External options: {} -- Drop external table 'exttabtest_r_newname' DROP EXTERNAL TABLE exttabtest_r_newname; -DROP EXTERNAL TABLE -- DROP EXTERNAL TABLE -- Test 38: Negative - handling invalid data formate when using custom protocol -- Recreate RET DROP EXTERNAL TABLE IF EXISTS exttabtest_r_invalid; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_invalid" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_r_invalid" does not exist, skipping CREATE READABLE EXTERNAL TABLE exttabtest_r_invalid(like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- Using WET to create data source file exttabtest.txt SELECT * FROM clean_exttabtest_files; stdout @@ -779,7 +699,6 @@ CREATE EXTERNAL TABLE (0 rows) INSERT INTO exttabtest_w (select * from exttabtest); -INSERT 0 100 -- Check RET is accessible SELECT count(*) from exttabtest_r_invalid; count @@ -790,38 +709,32 @@ INSERT 0 100 -- returns count = 100 -- Add a column (value3) to RET ALTER EXTERNAL TABLE exttabtest_r_invalid ADD COLUMN value3 int; -ALTER EXTERNAL TABLE -- Access RET again with changed structure, the data file format is invalid now SELECT count(*) from exttabtest_r_invalid; -psql:/path/sql_file:1: ERROR: missing data for column "value3" (seg0 slice1 rh55-qavm55:7532 pid=14575) +ERROR: missing data for column "value3" (seg0 slice1 rh55-qavm55:7532 pid=14575) DETAIL: External table exttabtest_r_invalid, line 1 of demoprot://exttabtest.txt: "1 name1 2 3" -- Drop a column (id) from RET ALTER EXTERNAL TABLE exttabtest_r_invalid DROP COLUMN id; -ALTER EXTERNAL TABLE -- Access RET again with changed structure, the data file format is also invalid -- Comment this out since output file is undeterministic -- SELECT count(*) from exttabtest_r_invalid where gp_segment_id=0; -- Recreate WET DROP EXTERNAL TABLE IF EXISTS exttabtest_w_invalid; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_invalid" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_w_invalid" does not exist, skipping CREATE WRITABLE EXTERNAL TABLE exttabtest_w_invalid(like exttabtest) LOCATION('demoprot://exttabtest_invalid.txt') FORMAT 'text'; -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -- Drop a column (id) from WET ALTER EXTERNAL TABLE exttabtest_w_invalid DROP COLUMN id; -psql:/path/sql_file:1: NOTICE: Dropping a column that is part of the distribution policy forces a NULL distribution policy -ALTER EXTERNAL TABLE +NOTICE: Dropping a column that is part of the distribution policy forces a NULL distribution policy -- Write to WET INSERT INTO exttabtest_w_invalid (SELECT * from exttabtest); -psql:/path/sql_file:1: ERROR: INSERT has more expressions than target columns +ERROR: INSERT has more expressions than target columns -- Test 60: setup source table -- This table is our example database table for formatter test DROP TABLE IF EXISTS formatsource CASCADE; -psql:/path/sql_file:1: NOTICE: table "formatsource" does not exist, skipping -DROP TABLE +NOTICE: table "formatsource" does not exist, skipping CREATE TABLE formatsource( name varchar(40), id float8, @@ -829,19 +742,19 @@ CREATE TABLE formatsource( value2 float8 ) DISTRIBUTED BY (id); -CREATE TABLE -- Loading 100 records \echo 'loading data...' loading data... INSERT INTO formatsource SELECT 'name'||i, i, i*2, i*3 FROM generate_series(1,100) i; -INSERT 0 100 -- Check data distribution - SELECT gp_segment_id, count(*) FROM formatsource GROUP BY 1 ORDER BY 1; - gp_segment_id | count ----------------+------- - 0 | 42 - 1 | 58 -(2 rows) +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from formatsource group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; + ?column? +---------- + f +(1 row) -- Test 61: create STABLE read and write functions based on example gpformatter.so -- Note: Only STABLE is supported for formatter. @@ -850,27 +763,21 @@ INSERT 0 100 CREATE OR REPLACE FUNCTION formatter_export_s(record) RETURNS bytea AS '$libdir/gpformatter.so', 'formatter_export' LANGUAGE C STABLE; -CREATE FUNCTION CREATE OR REPLACE FUNCTION formatter_import_s() RETURNS record AS '$libdir/gpformatter.so', 'formatter_import' LANGUAGE C STABLE; -CREATE FUNCTION CREATE OR REPLACE FUNCTION formatter_export_v(record) RETURNS bytea AS '$libdir/gpformatter.so', 'formatter_export' LANGUAGE C VOLATILE; -CREATE FUNCTION CREATE OR REPLACE FUNCTION formatter_import_v() RETURNS record AS '$libdir/gpformatter.so', 'formatter_import' LANGUAGE C VOLATILE; -CREATE FUNCTION CREATE OR REPLACE FUNCTION formatter_export_i(record) RETURNS bytea AS '$libdir/gpformatter.so', 'formatter_export' LANGUAGE C IMMUTABLE; -CREATE FUNCTION CREATE OR REPLACE FUNCTION formatter_import_i() RETURNS record AS '$libdir/gpformatter.so', 'formatter_import' LANGUAGE C IMMUTABLE; -CREATE FUNCTION -- Check pg_proc catalog table for new created functions SELECT proname, prolang,proisstrict,provolatile,pronargs,prorettype,prosrc,proacl FROM pg_proc WHERE proname like 'formatter%' @@ -889,15 +796,11 @@ CREATE FUNCTION CREATE OR REPLACE FUNCTION formatter_export_todrop(record) RETURNS bytea AS '$libdir/gpformatter.so', 'formatter_export' LANGUAGE C STABLE; -CREATE FUNCTION CREATE OR REPLACE FUNCTION formatter_import_todrop() RETURNS record AS '$libdir/gpformatter.so', 'formatter_import' LANGUAGE C STABLE; -CREATE FUNCTION DROP FUNCTION formatter_export_todrop(record); -DROP FUNCTION DROP FUNCTION formatter_import_todrop(); -DROP FUNCTION -- Test 63: create RET and WET using demoprot protocol and STABLE formatter -- Note: Only STABLE is supported for formatter, this is enforced -- When it is not STABLE (VOLATILE, or IMMUTABLE), @@ -906,37 +809,29 @@ DROP FUNCTION -- Create RET and WET using IMMUTABLE functions will succeed -- However query such RET or WET should fail DROP EXTERNAL TABLE IF EXISTS format_w; -psql:/path/sql_file:1: NOTICE: table "format_w" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "format_w" does not exist, skipping CREATE WRITABLE EXTERNAL TABLE format_w(like formatsource) LOCATION ('demoprot://exttabtest_test63') FORMAT 'CUSTOM' (FORMATTER='formatter_export_i'); -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table DROP EXTERNAL TABLE IF EXISTS format_r; -psql:/path/sql_file:1: NOTICE: table "format_r" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "format_r" does not exist, skipping CREATE READABLE EXTERNAL TABLE format_r(like formatsource) LOCATION ('demoprot://exttabtest_test63') FORMAT 'CUSTOM' (FORMATTER='formatter_import_i'); -CREATE EXTERNAL TABLE INSERT INTO format_w (SELECT * FROM formatsource); -psql:/path/sql_file:1: ERROR: formatter function formatter_export_i is not declared STABLE. (seg1 rh55-qavm55:7533 pid=30568) +ERROR: formatter function formatter_export_i is not declared STABLE. (seg1 rh55-qavm55:7533 pid=30568) -- ERROR: formatter function formatter_export_i is not declared STABLE. (seg1 rh55-qavm55:7533 pid=14816) -- Create RET and WET using STABLE functions DROP EXTERNAL TABLE IF EXISTS format_w; -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE format_w(like formatsource) LOCATION ('demoprot://exttabtest_test63') FORMAT 'CUSTOM' (FORMATTER='formatter_export_s'); -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table DROP EXTERNAL TABLE IF EXISTS format_r; -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE format_r(like formatsource) LOCATION ('demoprot://exttabtest_test63') FORMAT 'CUSTOM' (FORMATTER='formatter_import_s'); -CREATE EXTERNAL TABLE -- Displaying table info \d format_w External table "exttableext.format_w" @@ -993,7 +888,6 @@ External location: demoprot://exttabtest_test63 (0 rows) INSERT INTO format_w (SELECT * FROM formatsource); -INSERT 0 100 -- read from RET SELECT * FROM format_r EXCEPT ALL @@ -1010,23 +904,20 @@ INSERT 0 100 (1 row) -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from format_r - GROUP BY 1 ORDER BY 1; - gp_segment_id | count ----------------+------- - 0 | 42 - 1 | 58 -(2 rows) +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from format_r group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; + ?column? +---------- + f +(1 row) -- Test 64: Drop format function with external table using the function DROP FUNCTION formatter_export_i(record); -DROP FUNCTION DROP FUNCTION formatter_import_i(); -DROP FUNCTION DROP FUNCTION formatter_export_s(record); -DROP FUNCTION DROP FUNCTION formatter_import_s(); -DROP FUNCTION -- Check pg_proc catalog table for the format functions -- should return 0 SELECT proname, prolang,proisstrict,provolatile, @@ -1058,21 +949,19 @@ DROP FUNCTION (0 rows) INSERT INTO format_w (SELECT * FROM formatsource); -psql:/path/sql_file:1: ERROR: formatter function formatter_export_s of type writable was not found. (seg0 rh55-qavm55:7532 pid=30616) +ERROR: formatter function formatter_export_s of type writable was not found. (seg0 rh55-qavm55:7532 pid=30616) -- Read from RET, should fail SELECT count(*) FROM format_r; -psql:/path/sql_file:1: ERROR: formatter function formatter_import_s of type readable was not found. +ERROR: formatter function formatter_import_s of type readable was not found. HINT: Create it with CREATE FUNCTION. -- Test 65: Restore (recreate) functions with same name, external table should work again -- Recreate UDFs with same function names CREATE OR REPLACE FUNCTION formatter_export_s(record) RETURNS bytea AS '$libdir/gpformatter.so', 'formatter_export' LANGUAGE C STABLE; -CREATE FUNCTION CREATE OR REPLACE FUNCTION formatter_import_s() RETURNS record AS '$libdir/gpformatter.so', 'formatter_import' LANGUAGE C STABLE; -CREATE FUNCTION -- Check pg_proc catalog table for new created functions SELECT proname, prolang,proisstrict,provolatile, pronargs,prorettype,prosrc,proacl FROM pg_proc @@ -1091,7 +980,6 @@ CREATE FUNCTION (0 rows) INSERT INTO format_w (SELECT * FROM formatsource); -INSERT 0 100 -- Read from RET SELECT count(*) FROM format_r; count @@ -1105,45 +993,35 @@ INSERT 0 100 CREATE OR REPLACE FUNCTION formatter_export_s(record) RETURNS bytea AS '$libdir/gpformatter.so', 'formatter_export' LANGUAGE C STABLE; -CREATE FUNCTION CREATE OR REPLACE FUNCTION formatter_import_s() RETURNS record AS '$libdir/gpformatter.so', 'formatter_import' LANGUAGE C STABLE; -CREATE FUNCTION -- First pair of RET and WET that using -- formatter_import_s and formatter_export_s DROP EXTERNAL TABLE IF EXISTS format_w_s1; -psql:/path/sql_file:1: NOTICE: table "format_w_s1" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "format_w_s1" does not exist, skipping CREATE WRITABLE EXTERNAL TABLE format_w_s1(like formatsource) LOCATION ('demoprot://exttabtest_test67_s1') FORMAT 'CUSTOM' (FORMATTER='formatter_export_s'); -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table DROP EXTERNAL TABLE IF EXISTS format_r_s1; -psql:/path/sql_file:1: NOTICE: table "format_r_s1" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "format_r_s1" does not exist, skipping CREATE READABLE EXTERNAL TABLE format_r_s1(like formatsource) LOCATION ('demoprot://exttabtest_test67_s1') FORMAT 'CUSTOM' (FORMATTER='formatter_import_s'); -CREATE EXTERNAL TABLE -- Second pair of RET and WET that using -- same formatter_import_s and formatter_export_s DROP EXTERNAL TABLE IF EXISTS format_w_s2; -psql:/path/sql_file:1: NOTICE: table "format_w_s2" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "format_w_s2" does not exist, skipping CREATE WRITABLE EXTERNAL TABLE format_w_s2(like formatsource) LOCATION ('demoprot://exttabtest_test67_s2') FORMAT 'CUSTOM' (FORMATTER='formatter_export_s'); -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table DROP EXTERNAL TABLE IF EXISTS format_r_s2; -psql:/path/sql_file:1: NOTICE: table "format_r_s2" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "format_r_s2" does not exist, skipping CREATE READABLE EXTERNAL TABLE format_r_s2(like formatsource) LOCATION ('demoprot://exttabtest_test67_s2') FORMAT 'CUSTOM' (FORMATTER='formatter_import_s'); -CREATE EXTERNAL TABLE -- Checking pg_exttable select pg_class.relname, fmttype, fmtopts, encoding, writable @@ -1162,9 +1040,7 @@ CREATE EXTERNAL TABLE -- Write to WET INSERT INTO format_w_s1 (SELECT * FROM formatsource); -INSERT 0 100 INSERT INTO format_w_s2 (SELECT * FROM formatsource); -INSERT 0 100 -- read from RET SELECT * FROM format_r_s1 EXCEPT ALL @@ -1196,13 +1072,11 @@ INSERT 0 100 -- Test 71: Check limit of MAX_FORMAT_STRING 4096 bytes for variable length strings - text type -- Create format_long table for formatter long record test DROP TABLE IF EXISTS format_long CASCADE; -psql:/path/sql_file:1: NOTICE: table "format_long" does not exist, skipping -DROP TABLE +NOTICE: table "format_long" does not exist, skipping CREATE TABLE format_long ( id float8, name text ) DISTRIBUTED by (id); -CREATE TABLE -- Check the atttypmod of column name is -1 SELECT atttypmod FROM pg_attribute WHERE attrelid = 'format_long'::regclass @@ -1215,80 +1089,67 @@ CREATE TABLE -- returns atttypmod = -1 -- Create RET and WET using demoprot and custom format UDFs DROP EXTERNAL TABLE IF EXISTS format_long_w; -psql:/path/sql_file:1: NOTICE: table "format_long_w" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "format_long_w" does not exist, skipping CREATE WRITABLE EXTERNAL TABLE format_long_w(like format_long) LOCATION ('demoprot://exttabtest_test71.txt') FORMAT 'CUSTOM' (FORMATTER='formatter_export_s'); -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table DROP EXTERNAL TABLE IF EXISTS format_long_r; -psql:/path/sql_file:1: NOTICE: table "format_long_r" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "format_long_r" does not exist, skipping CREATE READABLE EXTERNAL TABLE format_long_r(like format_long) LOCATION ('demoprot://exttabtest_test71.txt') FORMAT 'CUSTOM' (FORMATTER='formatter_import_s'); -CREATE EXTERNAL TABLE -- Loading each record with 4000 characters, less than 4096 bytes truncate table format_long; -TRUNCATE TABLE insert into format_long select 1.0,i from repeat('oxo1', 1000) i; -INSERT 0 1 insert into format_long select 2.0,i from repeat('oxo2', 1000) i; -INSERT 0 1 insert into format_long select 3.0,i from repeat('oxo3', 1000) i; -INSERT 0 1 insert into format_long select 4.0,i from repeat('oxo4', 1000) i; -INSERT 0 1 + insert into format_long select 5.0,i from repeat('oxo5', 1000) i; + insert into format_long select 6.0,i from repeat('oxo6', 1000) i; + insert into format_long select 7.0,i from repeat('oxo7', 1000) i; + insert into format_long select 8.0,i from repeat('oxo8', 1000) i; + insert into format_long select 9.0,i from repeat('oxo9', 1000) i; -- Check distribution is even - select gp_segment_id,count(*) from format_long - group by gp_segment_id - order by gp_segment_id; - gp_segment_id | count ----------------+------- - 0 | 1 - 1 | 3 -(2 rows) +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from format_long group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; + ?column? +---------- + f +(1 row) -- Write to WET -- insert should be successful INSERT INTO format_long_w (SELECT * FROM format_long); -INSERT 0 4 - -- INSERT 0 4 -- Read from RET select count(*) from format_long_r; count ------- - 4 + 9 (1 row) -- returns count = 4 -- Now loading each record with 5000 characters, more than 4096 bytes for each record truncate table format_long; -TRUNCATE TABLE insert into format_long select 1.0,i from repeat('oxox1', 1000) i; -INSERT 0 1 insert into format_long select 2.0,i from repeat('oxox2', 1000) i; -INSERT 0 1 insert into format_long select 3.0,i from repeat('oxox3', 1000) i; -INSERT 0 1 insert into format_long select 4.0,i from repeat('oxox4', 1000) i; -INSERT 0 1 -- Write to WET -- insert should fail since MAX_FORMAT_STRING (4096) is exceeded INSERT INTO format_long_w (SELECT * FROM format_long); -psql:/path/sql_file:1: ERROR: formatter_export: buffer too small (gpformatter.c:182) (seg0 rh55-qavm55:7532 pid=22752) (cdbdisp.c:1453) +ERROR: formatter_export: buffer too small (gpformatter.c:182) (seg0 rh55-qavm55:7532 pid=22752) (cdbdisp.c:1453) -- ERROR: formatter_export: buffer too small (gpformatter.c:182) (seg2 rh55-qavm58:5532 pid=20093) -- Test 72: Verify limit of MAX_FORMAT_STRING 4096 bytes does not apply to fixed length strings - varchar() -- MAX_FORMAT_STRING limit should not apply to fixed length strings (char or varchar) -- Create format_long table for formatter long record test DROP TABLE IF EXISTS format_long CASCADE; -DROP TABLE CREATE TABLE format_long ( id float8, name varchar(6000) ) DISTRIBUTED by (id); -CREATE TABLE -- Check the atttypmod of name > 0 SELECT atttypmod FROM pg_attribute WHERE attrelid = 'format_long'::regclass @@ -1301,53 +1162,45 @@ CREATE TABLE -- returns atttypmod = 6004 -- Create RET and WET using demoprot and custom format UDFs DROP EXTERNAL TABLE IF EXISTS format_long_w; -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE format_long_w(like format_long) LOCATION ('demoprot://exttabtest_test72.txt') FORMAT 'CUSTOM' (FORMATTER='formatter_export_s'); -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table DROP EXTERNAL TABLE IF EXISTS format_long_r; -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE format_long_r(like format_long) LOCATION ('demoprot://exttabtest_test72.txt') FORMAT 'CUSTOM' (FORMATTER='formatter_import_s'); -CREATE EXTERNAL TABLE -- Now loading each record with 5000 characters, more than 4096 bytes for each record truncate table format_long; -TRUNCATE TABLE insert into format_long select 1.0,i from repeat('oxox1', 1000) i; -INSERT 0 1 insert into format_long select 2.0,i from repeat('oxox2', 1000) i; -INSERT 0 1 insert into format_long select 3.0,i from repeat('oxox3', 1000) i; -INSERT 0 1 insert into format_long select 4.0,i from repeat('oxox4', 1000) i; -INSERT 0 1 + insert into format_long select 5.0,i from repeat('oxox5', 1000) i; + insert into format_long select 6.0,i from repeat('oxox6', 1000) i; + insert into format_long select 7.0,i from repeat('oxox7', 1000) i; + insert into format_long select 8.0,i from repeat('oxox8', 1000) i; + insert into format_long select 9.0,i from repeat('oxox9', 1000) i; -- Write to WET -- insert should be successful INSERT INTO format_long_w (SELECT * FROM format_long); -INSERT 0 4 - -- INSERT 0 4 -- Read from RET select count(*) from format_long_r; count ------- - 4 + 9 (1 row) - -- returns count = 4 + -- returns count = 9 -- Test 73: Interlacing short and long record, testing FMT_NEED_MORE_DATA -- When loading data from RET, long record may trigger FORMATTER_RETURN_NOTIFICATION(fcinfo, FMT_NEED_MORE_DATA). -- Verify the data can be loaded successfully and should exactly match the source records. -- Create format_long table for formatter long record test DROP TABLE IF EXISTS format_long CASCADE; -DROP TABLE CREATE TABLE format_long ( id float8, name varchar(5000) ) DISTRIBUTED by (id); -CREATE TABLE -- Check the atttypmod of name > 0 SELECT atttypmod FROM pg_attribute WHERE attrelid = 'format_long'::regclass @@ -1360,44 +1213,38 @@ CREATE TABLE -- returns atttypmod = 5004 -- Create RET and WET using demoprot and custom format UDFs DROP EXTERNAL TABLE IF EXISTS format_long_w; -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE format_long_w(like format_long) LOCATION ('demoprot://exttabtest_test73.txt') FORMAT 'CUSTOM' (FORMATTER='formatter_export_s'); -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table DROP EXTERNAL TABLE IF EXISTS format_long_r; -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE format_long_r(like format_long) LOCATION ('demoprot://exttabtest_test73.txt') FORMAT 'CUSTOM' (FORMATTER='formatter_import_s'); -CREATE EXTERNAL TABLE -- Loading short and long records alternatively. -- This should trigger FORMATTER_RETURN_NOTIFICATION(fcinfo, FMT_NEED_MORE_DATA) -- However not really sure about this. truncate table format_long; -TRUNCATE TABLE insert into format_long select 1.0,'oxo1'; -INSERT 0 1 insert into format_long select 2.0,i from repeat('oxo2', 1000) i; -INSERT 0 1 insert into format_long select 3.0,'oxo3'; -INSERT 0 1 insert into format_long select 4.0,i from repeat('oxo4', 1000) i; -INSERT 0 1 + insert into format_long select 5.0,'oxo5'; + insert into format_long select 6.0,i from repeat('oxo6', 1000) i; + insert into format_long select 7.0,'oxo7'; + insert into format_long select 8.0,i from repeat('oxo8', 1000) i; + insert into format_long select 9.0,'oxo9'; -- Write to WET -- insert should be successful INSERT INTO format_long_w (SELECT * FROM format_long); -INSERT 0 4 - -- INSERT 0 4 -- Read from RET select count(*) from format_long_r; count ------- - 4 + 9 (1 row) - -- returns count = 4 + -- returns count = 9 -- read from RET, both should return 0 SELECT * FROM format_long_r EXCEPT ALL @@ -1410,47 +1257,36 @@ INSERT 0 4 -- SAS example formatter only support String and Float types. For other types of data (like INT), it throws error "unsupported data type" -- Create format_long table for formatter long record test DROP TABLE IF EXISTS format_long CASCADE; -DROP TABLE CREATE TABLE format_long ( id int, name text ) DISTRIBUTED by (id); -CREATE TABLE -- Create RET and WET using demoprot and custom format UDFs DROP EXTERNAL TABLE IF EXISTS format_long_w; -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE format_long_w(like format_long) LOCATION ('demoprot://format_long_test14') FORMAT 'CUSTOM' (FORMATTER='formatter_export_s'); -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table DROP EXTERNAL TABLE IF EXISTS format_long_r; -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE format_long_r(like format_long) -- using source data file format_long_test13 created by previous test LOCATION ('demoprot://format_long_test13') FORMAT 'CUSTOM' (FORMATTER='formatter_import_s'); -CREATE EXTERNAL TABLE -- Now loading each record with 4000 characters, less than 4096 bytes for each record truncate table format_long; -TRUNCATE TABLE insert into format_long select 1,'oxo1'; -INSERT 0 1 insert into format_long select 2,'oxo2'; -INSERT 0 1 insert into format_long select 3,'oxo3'; -INSERT 0 1 insert into format_long select 4,'oxo4'; -INSERT 0 1 -- Write to WET -- insert should fail since INT type is not supported INSERT INTO format_long_w (SELECT * FROM format_long); -psql:/path/sql_file:1: ERROR: formatter_export error: unsupported data type (gpformatter.c:100) (seg1 rh55-qavm55:7533 pid=27134) (cdbdisp.c:1453) +ERROR: formatter_export error: unsupported data type (gpformatter.c:100) (seg1 rh55-qavm55:7533 pid=27134) (cdbdisp.c:1453) -- ERROR: formatter_export error: unsupported data type (gpformatter.c:100) (seg2 rh55-qavm58:5532 pid=20668) (cdbdisp.c:1458) -- Read from RET using data file format_long_test13 create by previous test -- select should fail since INT type is not supported select count(*) from format_long_r; -psql:/path/sql_file:1: ERROR: demoprot_import: could not open file "format_long_test13" for reading: No such file or directory (seg1 slice1 rh55-qavm55:7533 pid=27134) +ERROR: demoprot_import: could not open file "format_long_test13" for reading: No such file or directory (seg1 slice1 rh55-qavm55:7533 pid=27134) DETAIL: External table format_long_r -- ERROR: formatter_import error: unsupported data type (gpformatter.c:256) (seg1 slice1 rh55-qavm57:5533 pid=20204) (cdbdisp.c:1458) -- Test 75: External table contains dropped columns @@ -1458,55 +1294,41 @@ DETAIL: External table format_long_r -- Both import and export operations will fail. This is expected. -- Create example source table formatsource DROP TABLE IF EXISTS formatsource CASCADE; -DROP TABLE CREATE TABLE formatsource ( id float8, name text, value1 float8 ) DISTRIBUTED by (id); -CREATE TABLE -- Create RET and WET using demoprot and custom format UDFs DROP EXTERNAL TABLE IF EXISTS format_w; -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE format_w(like formatsource) LOCATION ('demoprot://format_test15') FORMAT 'CUSTOM' (FORMATTER='formatter_export_s'); -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table DROP EXTERNAL TABLE IF EXISTS format_r; -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE format_r(like formatsource) -- using source data file format_long_test13 created by previous test LOCATION ('demoprot://format_long_test13') FORMAT 'CUSTOM' (FORMATTER='formatter_import_s'); -CREATE EXTERNAL TABLE -- Drop colum value1 from source table formatsource ALTER TABLE formatsource DROP COLUMN value1; -ALTER TABLE -- Drop colum value1 from RET and WET ALTER EXTERNAL TABLE format_r DROP COLUMN value1; -ALTER EXTERNAL TABLE ALTER EXTERNAL TABLE format_w DROP COLUMN value1; -ALTER EXTERNAL TABLE -- Loading records truncate table formatsource; -TRUNCATE TABLE insert into formatsource select 1,'oxo1'; -INSERT 0 1 insert into formatsource select 2,'oxo2'; -INSERT 0 1 insert into formatsource select 3,'oxo3'; -INSERT 0 1 insert into formatsource select 4,'oxo4'; -INSERT 0 1 -- Write to WET -- insert should failed because of dropped column value1 INSERT INTO format_w (SELECT * FROM formatsource); -psql:/path/sql_file:1: ERROR: formatter_export: dropped columns (gpformatter.c:80) (seg0 rh55-qavm55:7532 pid=27212) (cdbdisp.c:1453) +ERROR: formatter_export: dropped columns (gpformatter.c:80) (seg0 rh55-qavm55:7532 pid=27212) (cdbdisp.c:1453) -- ERROR: formatter_export: dropped columns (gpformatter.c:80) (seg1 rh55-qavm57:5533 pid=20454) (cdbdisp.c:1458) -- Read from RET using data file format_long_test13 create by previous test select count(*) from format_r; -psql:/path/sql_file:1: ERROR: demoprot_import: could not open file "format_long_test13" for reading: No such file or directory (seg0 slice1 rh55-qavm55:7532 pid=27212) +ERROR: demoprot_import: could not open file "format_long_test13" for reading: No such file or directory (seg0 slice1 rh55-qavm55:7532 pid=27212) DETAIL: External table format_r -- ERROR: formatter_import: dropped columns (gpformatter.c:244) (seg2 slice1 rh55-qavm58:5532 pid=20911) (cdbdisp.c:1458) -- Test 76: External table contains added column @@ -1514,47 +1336,37 @@ DETAIL: External table format_r -- The value of the added column would be either null or NaN, depends on the data type. -- Create example source table formatsource DROP TABLE IF EXISTS formatsource CASCADE; -DROP TABLE CREATE TABLE formatsource ( id float8, name char(10) ) DISTRIBUTED by (id); -CREATE TABLE -- Create RET and WET using demoprot and custom format UDFs DROP EXTERNAL TABLE IF EXISTS format_w; -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE format_w(like formatsource) LOCATION ('demoprot://exttabtest_test76.txt') FORMAT 'CUSTOM' (FORMATTER='formatter_export_s'); -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table DROP EXTERNAL TABLE IF EXISTS format_r; -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE format_r(like formatsource) LOCATION ('demoprot://exttabtest_test76.txt') FORMAT 'CUSTOM' (FORMATTER='formatter_import_s'); -CREATE EXTERNAL TABLE -- Add column value1 to RET and WET ALTER EXTERNAL TABLE format_r ADD COLUMN value1 float8; -ALTER EXTERNAL TABLE ALTER EXTERNAL TABLE format_w ADD COLUMN value1 float8; -ALTER EXTERNAL TABLE -- Loading records truncate table formatsource; -TRUNCATE TABLE insert into formatsource select 1,'oxo1'; -INSERT 0 1 insert into formatsource select 2,'oxo2'; -INSERT 0 1 insert into formatsource select 3,'oxo3'; -INSERT 0 1 insert into formatsource select 4,'oxo4'; -INSERT 0 1 + insert into formatsource select 5,'oxo5'; + insert into formatsource select 6,'oxo6'; + insert into formatsource select 7,'oxo7'; + insert into formatsource select 8,'oxo8'; + insert into formatsource select 9,'oxo9'; -- Write to WET -- insert should be successful INSERT INTO format_w (SELECT * FROM formatsource); -INSERT 0 4 - -- INSERT 0 4 -- Read from RET select id, value1 from format_r order by id; id | value1 @@ -1563,7 +1375,12 @@ INSERT 0 4 2 | NaN 3 | NaN 4 | NaN -(4 rows) + 5 | NaN + 6 | NaN + 7 | NaN + 8 | NaN + 9 | NaN +(9 rows) -- Test 77: data with null values -- SAS example formatter casts null value to: @@ -1571,43 +1388,35 @@ INSERT 0 4 -- * NaN for Float8 -- Create example source table formatsource DROP TABLE IF EXISTS formatsource CASCADE; -DROP TABLE CREATE TABLE formatsource ( id float8, name text, value1 float8 ) DISTRIBUTED by (id); -CREATE TABLE -- Create RET and WET using demoprot and custom format UDFs DROP EXTERNAL TABLE IF EXISTS format_w; -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE format_w(like formatsource) LOCATION ('demoprot://exttabtest_test77.txt') FORMAT 'CUSTOM' (FORMATTER='formatter_export_s'); -psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table -CREATE EXTERNAL TABLE +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table DROP EXTERNAL TABLE IF EXISTS format_r; -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE format_r(like formatsource) LOCATION ('demoprot://exttabtest_test77.txt') FORMAT 'CUSTOM' (FORMATTER='formatter_import_s'); -CREATE EXTERNAL TABLE -- Loading records truncate table formatsource; -TRUNCATE TABLE insert into formatsource values (1,null,null); -INSERT 0 1 insert into formatsource values (2,null,null); -INSERT 0 1 insert into formatsource values (3,null,null); -INSERT 0 1 insert into formatsource values (4,null,null); -INSERT 0 1 + insert into formatsource values (5,null,null); + insert into formatsource values (6,null,null); + insert into formatsource values (7,null,null); + insert into formatsource values (8,null,null); + insert into formatsource values (9,null,null); -- Write to WET -- insert should be successful INSERT INTO format_w (SELECT * FROM formatsource); -INSERT 0 4 - -- INSERT 0 4 -- Read from RET select * from format_r where name is null order by id; id | name | value1 @@ -1616,26 +1425,27 @@ INSERT 0 4 2 | | NaN 3 | | NaN 4 | | NaN -(4 rows) + 5 | | NaN + 6 | | NaN + 7 | | NaN + 8 | | NaN + 9 | | NaN +(9 rows) -- Test 78: Read from empty data file -- SAS example formatter can correctly handle empty input, either from /dev/null or from empty input data files. -- Create example source table formatsource DROP TABLE IF EXISTS formatsource CASCADE; -DROP TABLE CREATE TABLE formatsource ( id float8, name text, value1 float8 ) DISTRIBUTED by (id); -CREATE TABLE -- Create RET with custom format using /dev/null as source file DROP EXTERNAL TABLE IF EXISTS format_r; -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE format_r(like formatsource) LOCATION ('demoprot:///dev/null') FORMAT 'CUSTOM' (FORMATTER='formatter_import_s'); -CREATE EXTERNAL TABLE -- Read from RET using /dev/null select * from format_r; id | name | value1 @@ -1646,14 +1456,11 @@ CREATE EXTERNAL TABLE -- create the protocol read and write STABLE functions CREATE OR REPLACE FUNCTION write_to_file() RETURNS integer AS '$libdir/gpextprotocol.so', 'demoprot_export' LANGUAGE C STABLE; -CREATE FUNCTION CREATE OR REPLACE FUNCTION read_from_file() RETURNS integer AS '$libdir/gpextprotocol.so', 'demoprot_import' LANGUAGE C STABLE; -CREATE FUNCTION -- create validation STABLE function CREATE OR REPLACE FUNCTION url_validator() RETURNS void AS '$libdir/gpextprotocol.so', 'demoprot_validate_urls' LANGUAGE C STABLE; -CREATE FUNCTION -- declare the protocol name along with in/out funcs and validator func DROP PROTOCOL IF EXISTS demoprot CASCADE; NOTICE: drop cascades to external table format_r @@ -1666,13 +1473,12 @@ NOTICE: drop cascades to external table format_r_s1 NOTICE: drop cascades to external table format_w_s1 NOTICE: drop cascades to external table exttabtest_w_invalid NOTICE: drop cascades to external table exttabtest_r_invalid -DROP PROTOCOL +NOTICE: drop cascades to external table exttabtest_w CREATE PROTOCOL demoprot ( readfunc = read_from_file, writefunc = write_to_file, validatorfunc = url_validator ); -CREATE PROTOCOL -- Test 82: At ext table create time, validate number of URLs cannot exceed number of primary segments CREATE READABLE EXTERNAL TABLE exttabtest_5url_r(like exttabtest) LOCATION('demoprot://test1.txt', @@ -1681,96 +1487,79 @@ CREATE PROTOCOL 'demoprot://test4.txt', 'demoprot://test5.txt') FORMAT 'text'; -psql:/path/sql_file:1: ERROR: more than 2 urls aren't allowed in this protocol +ERROR: more than 2 urls aren't allowed in this protocol -- ERROR: more than 2 urls aren't allowed in this protocol -- Test 83: At ext table create time, url string cannot contain "secured_directory" CREATE READABLE EXTERNAL TABLE exttabtest_3url_r(like exttabtest) LOCATION('demoprot://test1.txt', 'demoprot://secured_directory/test2.txt') FORMAT 'text'; -psql:/path/sql_file:1: ERROR: using 'secured_directory' in a url isn't allowed +ERROR: using 'secured_directory' in a url isn't allowed -- ERROR: using 'secured_directory' in a url isn't allowed -- Test 4: Negative - validator protocol function must return void -- create the validator function and returns integer, which is invalid DROP FUNCTION IF EXISTS url_validator(); -DROP FUNCTION CREATE OR REPLACE FUNCTION url_validator() RETURNS integer AS '$libdir/gpextprotocol.so', 'demoprot_validate_urls' LANGUAGE C STABLE; -CREATE FUNCTION -- declare the protocol name along with in/out funcs and validator func DROP PROTOCOL IF EXISTS demoprot CASCADE; -DROP PROTOCOL CREATE PROTOCOL demoprot ( readfunc = read_from_file, writefunc = write_to_file, validatorfunc = url_validator ); -psql:/path/sql_file:1: ERROR: validator protocol function url_validator() must return void +ERROR: validator protocol function url_validator() must return void -- Test 5: Negative - invalid protocol attribute name for validator function: must be "validatorfunc" -- declare the protocol using invalid attribute name "validatorproc" DROP PROTOCOL IF EXISTS demoprot CASCADE; -psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping -DROP PROTOCOL +NOTICE: protocol "demoprot" does not exist, skipping CREATE PROTOCOL demoprot ( readfunc = read_from_file, writefunc = write_to_file, validatorproc = url_validator ); -psql:/path/sql_file:1: ERROR: protocol attribute "validatorproc" not recognized +ERROR: protocol attribute "validatorproc" not recognized -- ERROR: protocol attribute "validatorproc" not recognized -- ERROR: using 'secured_directory' in a url isn't allowed -- Create multiple roles with login option so that they can be used for protocol permission tests and alter protocol tests -- Create another suerpuer user demoprot_super drop role if exists demoprot_super; -DROP ROLE create role demoprot_super with SUPERUSER LOGIN; -CREATE ROLE -- Create a non-privileged user demoprot_nopriv drop role if exists demoprot_nopriv; -DROP ROLE create role demoprot_nopriv with login ; NOTICE: resource queue required -- using default resource queue "pg_default" -CREATE ROLE -- Create a gphdfs_user with CREATEEXTTABLE privilege using gphdfs protocol drop role if exists gphdfs_user; -DROP ROLE create role gphdfs_user with login CREATEEXTTABLE (protocol='gphdfs'); -psql:/path/sql_file:1: WARNING: GRANT/REVOKE on gphdfs is deprecated +WARNING: GRANT/REVOKE on gphdfs is deprecated HINT: Issue the GRANT or REVOKE on the protocol itself -psql:/path/sql_file:1: NOTICE: resource queue required -- using default resource queue "pg_default" -CREATE ROLE +NOTICE: resource queue required -- using default resource queue "pg_default" -- WARNING: GRANT/REVOKE on gphdfs is deprecated -- HINT: Issue the GRANT or REVOKE on the protocol itself -- NOTICE: resource queue required -- using default resource queue "pg_default" -- CREATE ROLE -- Test 92: Rename existing protocol DROP FUNCTION IF EXISTS url_validator(); -DROP FUNCTION CREATE OR REPLACE FUNCTION url_validator() RETURNS void AS '$libdir/gpextprotocol.so', 'demoprot_validate_urls' LANGUAGE C STABLE; -CREATE FUNCTION CREATE PROTOCOL demoprot ( readfunc = read_from_file, writefunc = write_to_file, validatorfunc = url_validator ); -CREATE PROTOCOL -- Create external RET and WET DROP EXTERNAL TABLE IF EXISTS exttabtest_w; NOTICE: table "exttabtest_w" does not exist, skipping -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w(like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_r; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_r" does not exist, skipping CREATE READABLE EXTERNAL TABLE exttabtest_r(like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- write to WET SELECT * FROM clean_exttabtest_files; stdout @@ -1778,10 +1567,8 @@ CREATE EXTERNAL TABLE (0 rows) INSERT INTO exttabtest_w (SELECT * FROM exttabtest); -INSERT 0 100 -- Rename existing protocol ALTER PROTOCOL demoprot RENAME to demoprot_new; -ALTER PROTOCOL -- checking pg_extprotocol select ptcname, ptctrusted from pg_extprotocol @@ -1809,34 +1596,29 @@ External location: demoprot://exttabtest.txt -- Create a new ext table using the new protocol name DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_new" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_w_new" does not exist, skipping CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new(like exttabtest) LOCATION('demoprot_new://exttabtest.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_new" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_r_new" does not exist, skipping CREATE READABLE EXTERNAL TABLE exttabtest_r_new(like exttabtest) LOCATION('demoprot_new://exttabtest.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- Verify access old ext table that referencing old protocol name would fail -- This is expected. select * from exttabtest_r; -psql:/path/sql_file:1: ERROR: protocol "demoprot" does not exist (seg0 slice1 rh55-qavm55:7532 pid=18394) +ERROR: protocol "demoprot" does not exist (seg0 slice1 rh55-qavm55:7532 pid=18394) -- ERROR: protocol "demoprot" does not exist (seg1 slice1 rh55-qavm57:5533 pid=8558) -- Verify access new ext table would be successful -- However demoprot implementation prevents using any other protocol name than "demoprot" -- therefore the error is expected. select count(*) from exttabtest_r_new; -psql:/path/sql_file:1: ERROR: internal error: demoprot called with a different protocol (demoprot_new) (gpextprotocol.c:86) (seg0 slice1 rh55-qavm55:7532 pid=18394) (cdbdisp.c:1453) +ERROR: internal error: demoprot called with a different protocol (demoprot_new) (gpextprotocol.c:87) (seg0 slice1 rh55-qavm55:7532 pid=18394) (cdbdisp.c:1453) DETAIL: External table exttabtest_r_new, file demoprot_new://exttabtest.txt -- Rename protocol name back to demoprot ALTER PROTOCOL demoprot_new RENAME to demoprot; -ALTER PROTOCOL -- Verify access old ext table that referencing old protocol name would succeed select count(*) from exttabtest_r; count @@ -1854,12 +1636,10 @@ NOTICE: drop cascades to external table exttabtest_r_new NOTICE: drop cascades to external table exttabtest_w_new NOTICE: drop cascades to external table exttabtest_r NOTICE: drop cascades to external table exttabtest_w -DROP PROTOCOL CREATE TRUSTED PROTOCOL demoprot ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -CREATE PROTOCOL -- Check the owner of trusted protocl demoprot is current user select ptcname, ptctrusted from pg_extprotocol join pg_user @@ -1873,14 +1653,11 @@ CREATE PROTOCOL -- Change protocol demoprot owner to non-privileged user "demoprot_nopriv" ALTER PROTOCOL demoprot OWNER TO demoprot_nopriv; -ALTER PROTOCOL -- Drop the existing external RET and WET DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new; NOTICE: table "exttabtest_r_new" does not exist, skipping -DROP EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new; NOTICE: table "exttabtest_w_new" does not exist, skipping -DROP EXTERNAL TABLE -- Check the owner of demoprot is demoprot_nopriv -- and no protocol permission has been granted select ptcname, ptctrusted,ptcacl, usename @@ -1908,14 +1685,11 @@ DROP EXTERNAL TABLE -- to non-privileged user "demoprot_nopriv" so that this user -- can try to create external table using format: (like exttabtest) GRANT SELECT ON exttabtest TO demoprot_nopriv; -GRANT -- As superuser, REVOKE ALL privileges on protocol from owner demoprot_nopriv REVOKE ALL ON PROTOCOL demoprot FROM demoprot_nopriv; -REVOKE -- connect as non-privileged user "demoprot_nopriv" -- which is the owner of trusted demoprot SET ROLE demoprot_nopriv; -SET select user; current_user ----------------- @@ -1930,12 +1704,10 @@ SET CREATE READABLE EXTERNAL TABLE exttabtest_r_new (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new(like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE -- Verify non-privileged user "demoprot_nopriv" can export data via new created WET exttabtest_w_new SELECT * FROM clean_exttabtest_files; stdout @@ -1943,7 +1715,6 @@ CREATE EXTERNAL TABLE (0 rows) INSERT INTO exttabtest_w_new (SELECT * FROM exttabtest); -INSERT 0 100 -- Verify non-privileged user "demoprot_nopriv" can load data via new created RET exttabtest_r_new select count(*) from exttabtest_r_new; count @@ -1955,41 +1726,34 @@ INSERT 0 100 DROP PROTOCOL demoprot CASCADE; NOTICE: drop cascades to external table exttabtest_w_new NOTICE: drop cascades to external table exttabtest_r_new -DROP PROTOCOL RESET ROLE; -RESET -- Test 94: Untrusted protocol - Change ownership -- The owner of untrusted protocol (not a superuser) can still create external table -- using the untrusted protocol. - -- connect as superuser + -- connect as superuser -- create untrusted protocol demoprot DROP PROTOCOL IF EXISTS demoprot CASCADE; - psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping -DROP PROTOCOL CREATE PROTOCOL demoprot ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -CREATE PROTOCOL -- Try to change protocol demoprot owner to non-privileged user "demoprot_nopriv" -- Should get: ERROR: untrusted protocol "demoprot" can't be owned by non superuser -- Therefore the owner of trusted protocl demoprot is still the current superuser ALTER PROTOCOL demoprot OWNER TO demoprot_nopriv; -psql:/path/sql_file:1: ERROR: untrusted protocol "demoprot" can't be owned by non superuser +ERROR: untrusted protocol "demoprot" can't be owned by non superuser -- As superuser, GRANT SELECT permission on heap table "exttabtest" -- to non-privileged user "demoprot_nopriv" so that this user -- can try to create external table using format: (like exttabtest) GRANT SELECT ON exttabtest TO demoprot_nopriv; -GRANT -- As superuser, REVOKE ALL privileges on protocol from owner demoprot_nopriv -- The error is correctly shown REVOKE ALL ON PROTOCOL demoprot FROM demoprot_nopriv; -psql:/path/sql_file:1: ERROR: protocol "demoprot" is not trusted +ERROR: protocol "demoprot" is not trusted HINT: Only superusers may use untrusted protocols. -- ERROR: protocol "demoprot" is not trusted -- login as non-privileged user "demoprot_nopriv" SET ROLE demoprot_nopriv; -SET select user; current_user ----------------- @@ -2003,47 +1767,44 @@ SET CREATE READABLE EXTERNAL TABLE exttabtest_r_new2 (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text'; -psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot +ERROR: permission denied for external protocol demoprot CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new2 (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text' DISTRIBUTED BY (id); -psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot +ERROR: permission denied for external protocol demoprot -- Verified non superuser cannot drop the protocol DROP PROTOCOL demoprot; -psql:/path/sql_file:1: ERROR: must be owner of external protocol demoprot +ERROR: must be owner of external protocol demoprot -- Test 95: Alter protocol negative tests ALTER PROTOCOL demoprot ( readfunc = read_from_file_immutable, writefunc = write_to_file_immutable ); -psql:/path/sql_file:1: ERROR: syntax error at or near "(" +ERROR: syntax error at or near "(" LINE 1: ALTER PROTOCOL demoprot ( ^ -- ERROR: syntax error at or near "(" ALTER PROTOCOL demoprot update readfunc = read_from_file_immutable; -psql:/path/sql_file:1: ERROR: syntax error at or near "update" +ERROR: syntax error at or near "update" LINE 1: ALTER PROTOCOL demoprot update readfunc = read_from_file_imm... ^ -- ERROR: syntax error at or near "update" ALTER PROTOCOL demoprot trusted; -psql:/path/sql_file:1: ERROR: syntax error at or near "trusted" +ERROR: syntax error at or near "trusted" LINE 1: ALTER PROTOCOL demoprot trusted; ^ -- ERROR: syntax error at or near "trusted" RESET ROLE; -RESET -- Test 96: Untrusted protocol - Superuser -- Non-owner superuser does not have any limitation when using non-trusted protocol. -- login as superuser huangh5 -- create untrusted protocol demoprot DROP PROTOCOL IF EXISTS demoprot CASCADE; -DROP PROTOCOL CREATE PROTOCOL demoprot ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -CREATE PROTOCOL -- Check the owner of trusted protocl demoprot is current superuser select ptcname, ptctrusted from pg_extprotocol join pg_user @@ -2057,7 +1818,6 @@ CREATE PROTOCOL -- connect as a different superuser "demoprot_super" SET ROLE demoprot_super; -SET select user; current_user ---------------- @@ -2073,22 +1833,17 @@ SET -- Verify superuser "demoprot_super" can create new ext table using untrusted protocol demoprot DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new; NOTICE: table "exttabtest_r_new" does not exist, skipping -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE exttabtest_r_new(like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new; NOTICE: table "exttabtest_w_new" does not exist, skipping -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new(like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE -- Verify superuser demoprot_super can still export data via new created WET exttabtest_w_new INSERT INTO exttabtest_w_new (SELECT * FROM exttabtest); -INSERT 0 100 -- Verify superuser demoprot_super can access new created RET exttabtest_r_new select count(*) from exttabtest_r_new; count @@ -2097,41 +1852,32 @@ INSERT 0 100 (1 row) RESET ROLE; -RESET -- Test 97: Non-trusted protocol - non-priv user -- Non-privileged user cannot use non-trusted protocol to create external table. -- With granted permissions on existing external table, Non-privileged user can access existing WET and RET DROP EXTERNAL TABLE IF EXISTS exttabtest_r; NOTICE: table "exttabtest_r" does not exist, skipping -DROP EXTERNAL TABLE CREATE READABLE EXTERNAL TABLE exttabtest_r(like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_w; NOTICE: table "exttabtest_w" does not exist, skipping -DROP EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w(like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE -- As superuser, GRANT SELECT permission on RET -- and INSERT permission on WET to on-privileged user "demoprot_nopriv" -- to non-privileged user "demoprot_nopriv" GRANT SELECT ON exttabtest_r TO demoprot_nopriv; -GRANT GRANT INSERT ON exttabtest_w TO demoprot_nopriv; -GRANT -- As superuser, GRANT SELECT permission on heap table "exttabtest" -- to non-privileged user "demoprot_nopriv" so that this user -- can try to create external table using format: (like exttabtest) GRANT SELECT ON exttabtest TO demoprot_nopriv; -GRANT -- connect as non-privileged user "demoprot_nopriv" SET ROLE demoprot_nopriv; -SET select user; current_user ----------------- @@ -2167,7 +1913,6 @@ SET (0 rows) INSERT INTO exttabtest_w (SELECT * FROM exttabtest); -INSERT 0 100 -- Verify with SELECT permission granted, non-privileged user "demoprot_nopriv" -- can load data via RET that was created using non-trusted protocol select count(*) from exttabtest_r; @@ -2179,91 +1924,80 @@ INSERT 0 100 -- Verify non-privileged user "demoprot_nopriv" cannot create new ext table -- using untrusted protocol demoprot DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new2; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_new2" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_r_new2" does not exist, skipping CREATE READABLE EXTERNAL TABLE exttabtest_r_new2 (like exttabtest) LOCATION('demoprot://exttabtest.txt') FORMAT 'text'; -psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot +ERROR: permission denied for external protocol demoprot -- Test 98: Trusted Protocol - Negative Tests -- create protocol using incorrect keyword TRUST instead of TRUSTED DROP PROTOCOL IF EXISTS demoprot_trusted_bad; -psql:/path/sql_file:1: NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping -DROP PROTOCOL +NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping CREATE TRUST PROTOCOL demoprot_trusted_bad ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -psql:/path/sql_file:1: ERROR: syntax error at or near "TRUST" +ERROR: syntax error at or near "TRUST" LINE 1: CREATE TRUST PROTOCOL demoprot_trusted_bad ( ^ -- create protocol using incorrect keyword UNTRUSTED DROP PROTOCOL IF EXISTS demoprot_trusted_bad; -psql:/path/sql_file:1: NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping -DROP PROTOCOL +NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping CREATE UNTRUSTED PROTOCOL demoprot_trusted_bad ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -psql:/path/sql_file:1: ERROR: syntax error at or near "UNTRUSTED" +ERROR: syntax error at or near "UNTRUSTED" LINE 1: CREATE UNTRUSTED PROTOCOL demoprot_trusted_bad ( ^ -- create protocol using TRUSTED at wrong position DROP PROTOCOL IF EXISTS demoprot_trusted_bad; -psql:/path/sql_file:1: NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping -DROP PROTOCOL +NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping CREATE PROTOCOL TRUSTED demoprot_trusted_bad ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -psql:/path/sql_file:1: ERROR: syntax error at or near "demoprot_trusted_bad" +ERROR: syntax error at or near "demoprot_trusted_bad" LINE 1: CREATE PROTOCOL TRUSTED demoprot_trusted_bad ( ^ -- create protocol using TRUSTED at wrong position DROP PROTOCOL IF EXISTS demoprot_trusted_bad; -psql:/path/sql_file:1: NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping -DROP PROTOCOL +NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping CREATE PROTOCOL demoprot_trusted_bad TRUSTED ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -psql:/path/sql_file:1: ERROR: syntax error at or near "TRUSTED" +ERROR: syntax error at or near "TRUSTED" LINE 1: CREATE PROTOCOL demoprot_trusted_bad TRUSTED ( ^ RESET ROLE; -RESET -- Test 99: Trusted protocol - Grant All -- Grant all permissions ON trusted protocol to non-privileged user -- Non-privileged user can use trusted protocol to create external table. -- connect as superuser -- create trusted protocol demoprot DROP PROTOCOL IF EXISTS demoprot CASCADE; +NOTICE: drop cascades to external table exttabtest_w +NOTICE: drop cascades to external table exttabtest_r NOTICE: drop cascades to external table exttabtest_w_new NOTICE: drop cascades to external table exttabtest_r_new -DROP PROTOCOL CREATE TRUSTED PROTOCOL demoprot ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -CREATE PROTOCOL -- As superuser, GRANT SELECT permission on heap table "exttabtest" -- to non-privileged user "demoprot_nopriv" so that this user -- can try to create external table using format: (like exttabtest) GRANT SELECT ON exttabtest TO demoprot_nopriv; -GRANT -- Drop existing WET and RET DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new; NOTICE: table "exttabtest_r_new" does not exist, skipping -DROP EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new; NOTICE: table "exttabtest_w_new" does not exist, skipping -DROP EXTERNAL TABLE -- As superuser, GRANT ALL privileges on protocol to non-privileged user demoprot_nopriv GRANT ALL ON PROTOCOL demoprot TO demoprot_nopriv; -GRANT -- login as non-privileged user "demoprot_nopriv" SET ROLE demoprot_nopriv; -SET select user; current_user ----------------- @@ -2275,12 +2009,10 @@ SET CREATE READABLE EXTERNAL TABLE exttabtest_r_new (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE -- Verify non-privileged user "demoprot_nopriv" can export data via new created WET exttabtest_w_new SELECT * FROM clean_exttabtest_files; stdout @@ -2288,7 +2020,6 @@ CREATE EXTERNAL TABLE (0 rows) INSERT INTO exttabtest_w_new (SELECT * FROM exttabtest); -INSERT 0 100 -- Verify non-privileged user "demoprot_nopriv" can load data via new created RET exttabtest_r_new select count(*) from exttabtest_r_new; count @@ -2297,7 +2028,6 @@ INSERT 0 100 (1 row) RESET ROLE; -RESET -- Test 99a: Trusted protocol - Revoke All -- Revoke all permissions ON trusted protocol from non-privileged user -- connect as superuser @@ -2305,44 +2035,39 @@ RESET DROP PROTOCOL IF EXISTS demoprot CASCADE; NOTICE: drop cascades to external table exttabtest_w_new NOTICE: drop cascades to external table exttabtest_r_new -DROP PROTOCOL CREATE TRUSTED PROTOCOL demoprot ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -CREATE PROTOCOL -- As superuser, GRANT SELECT permission on heap table "exttabtest" -- to non-privileged user "demoprot_nopriv" so that this user -- can try to create external table using format: (like exttabtest) GRANT SELECT ON exttabtest TO demoprot_nopriv; -GRANT -- As superuser, REVOKE ALL privileges on protocol from non-privileged user demoprot_nopriv -- Both SELECT and INSERT permissions are revoked REVOKE ALL ON PROTOCOL demoprot FROM demoprot_nopriv; NOTICE: no privileges could be revoked from role demoprot_nopriv on object demoprot -REVOKE -- login as non-privileged user "demoprot_nopriv" SET ROLE demoprot_nopriv; -SET select user; current_user ----------------- demoprot_nopriv (1 row) + -- Verify after permissions have been revoked -- non-privileged user "demoprot_nopriv" cannot create new ext table -- using the trusted protocol demoprot CREATE READABLE EXTERNAL TABLE exttabtest_r_new (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text'; -psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot +ERROR: permission denied for external protocol demoprot CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text' DISTRIBUTED BY (id); -psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot +ERROR: permission denied for external protocol demoprot RESET ROLE; -RESET -- Test 101: Trusted protocol - Grant Select -- Grant SELECT permission ON trusted protocol to non-privileged user -- Non-privileged user can use trusted protocol to create readable external table. @@ -2351,41 +2076,31 @@ RESET LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE - SELECT * FROM clean_exttabtest_files; stdout -------- (0 rows) INSERT INTO exttabtest_w (SELECT * FROM exttabtest); -INSERT 0 100 -- As superuser, demoport is created as a trusted readonly protocol DROP PROTOCOL IF EXISTS demoprot CASCADE; NOTICE: drop cascades to external table exttabtest_w -DROP PROTOCOL CREATE TRUSTED PROTOCOL demoprot ( readfunc = read_from_file_stable ); -CREATE PROTOCOL -- As superuser, GRANT SELECT permission on heap table "exttabtest" -- to non-privileged user "demoprot_nopriv" so that this user -- can try to create external table using format: (like exttabtest) GRANT SELECT ON exttabtest TO demoprot_nopriv; -GRANT -- Drop existing WET and RET DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new; NOTICE: table "exttabtest_r_new" does not exist, skipping -DROP EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new; NOTICE: table "exttabtest_w_new" does not exist, skipping -DROP EXTERNAL TABLE -- As superuser, GRANT SELECT permission on read protocol to non-privileged user demoprot_nopriv GRANT SELECT ON PROTOCOL demoprot TO demoprot_nopriv; -GRANT -- login as non-privileged user "demoprot_nopriv" SET ROLE demoprot_nopriv; -SET select user; current_user ----------------- @@ -2397,13 +2112,12 @@ SET CREATE READABLE EXTERNAL TABLE exttabtest_r_new (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text'; -CREATE EXTERNAL TABLE -- Verify non-privileged user "demoprot_nopriv" cannot create new writable ext table CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text' DISTRIBUTED BY (id); -psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot +ERROR: permission denied for external protocol demoprot -- Verify non-privileged user "demoprot_nopriv" can load data via new created RET exttabtest_r_new select count(*) from exttabtest_r_new; count @@ -2412,31 +2126,25 @@ psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot (1 row) RESET ROLE; -RESET -- Test 102: Trusted protocol - Revoke Select -- Revoke SELECT permission ON trusted protocol from non-privileged user -- connect as superuser -- create trusted protocol demoprot DROP PROTOCOL IF EXISTS demoprot CASCADE; NOTICE: drop cascades to external table exttabtest_r_new -DROP PROTOCOL CREATE TRUSTED PROTOCOL demoprot ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -CREATE PROTOCOL -- As superuser, GRANT SELECT permission on heap table "exttabtest" -- to non-privileged user "demoprot_nopriv" so that this user -- can try to create external table using format: (like exttabtest) GRANT SELECT ON exttabtest TO demoprot_nopriv; -GRANT -- As superuser, REVOKE SElECT privilege on protocol from non-privileged user demoprot_nopriv REVOKE SELECT ON PROTOCOL demoprot FROM demoprot_nopriv; NOTICE: no privileges could be revoked from role demoprot_nopriv on object demoprot -REVOKE -- login as non-privileged user "demoprot_nopriv" SET ROLE demoprot_nopriv; -SET select user; current_user ----------------- @@ -2449,36 +2157,28 @@ SET CREATE READABLE EXTERNAL TABLE exttabtest_r_new2 (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text'; -psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot +ERROR: permission denied for external protocol demoprot RESET ROLE; -RESET -- Test 103: Trusted protocol - Grant Insert -- Grant INSERT permission ON trusted protocol to non-privileged user -- Non-privileged user can use trusted protocol to create writable external table. -- As superuser, demoport is created as a trusted readonly protocol DROP PROTOCOL IF EXISTS demoprot CASCADE; -DROP PROTOCOL CREATE TRUSTED PROTOCOL demoprot ( writefunc = write_to_file_stable ); -CREATE PROTOCOL -- As superuser, GRANT SELECT permission on heap table "exttabtest" -- to non-privileged user "demoprot_nopriv" so that this user -- can try to create external table using format: (like exttabtest) GRANT SELECT ON exttabtest TO demoprot_nopriv; -GRANT -- Drop existing WET and RET DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new; -DROP EXTERNAL TABLE DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_new" does not exist, skipping -DROP EXTERNAL TABLE +NOTICE: table "exttabtest_w_new" does not exist, skipping -- As superuser, GRANT INSERT permission on read protocol to non-privileged user demoprot_nopriv GRANT INSERT ON PROTOCOL demoprot TO demoprot_nopriv; -GRANT -- login as non-privileged user "demoprot_nopriv" SET ROLE demoprot_nopriv; -SET select user; current_user ----------------- @@ -2490,13 +2190,12 @@ SET CREATE READABLE EXTERNAL TABLE exttabtest_r_new (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text'; -psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot +ERROR: permission denied for external protocol demoprot -- Verify non-privileged user "demoprot_nopriv" can create new writable ext table CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new (like exttabtest) LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text' DISTRIBUTED BY (id); -CREATE EXTERNAL TABLE -- Verify non-privileged user "demoprot_nopriv" can export data via new created WET exttabtest_w_new SELECT * FROM clean_exttabtest_files; stdout @@ -2504,32 +2203,26 @@ CREATE EXTERNAL TABLE (0 rows) INSERT INTO exttabtest_w_new (SELECT * FROM exttabtest); -INSERT 0 100 RESET ROLE; -RESET -- Test 104: Trusted protocol - Revoke Insert -- Revoke INSERT permission ON trusted protocol from non-privileged user -- connect as superuser -- create trusted protocol demoprot DROP PROTOCOL IF EXISTS demoprot CASCADE; NOTICE: drop cascades to external table exttabtest_w_new -DROP PROTOCOL CREATE TRUSTED PROTOCOL demoprot ( readfunc = read_from_file_stable, writefunc = write_to_file_stable ); -CREATE PROTOCOL -- As superuser, GRANT SELECT permission on heap table "exttabtest" -- to non-privileged user "demoprot_nopriv" so that this user -- can try to create external table using format: (like exttabtest) GRANT SELECT ON exttabtest TO demoprot_nopriv; -GRANT -- As superuser, REVOKE INSERT privilege on protocol from non-privileged user demoprot_nopriv REVOKE INSERT ON PROTOCOL demoprot FROM demoprot_nopriv; -REVOKE +NOTICE: no privileges could be revoked from role demoprot_nopriv on object demoprot -- login as non-privileged user "demoprot_nopriv" SET ROLE demoprot_nopriv; -SET select user; current_user ----------------- @@ -2543,6 +2236,5 @@ SET LOCATION('demoprot://exttabtest_new.txt') FORMAT 'text' DISTRIBUTED BY (id); -psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot +ERROR: permission denied for external protocol demoprot RESET ROLE; -RESET diff --git a/contrib/extprotocol/expected/setup.out b/contrib/extprotocol/expected/setup.out new file mode 100644 index 0000000000..340f2500c7 --- /dev/null +++ b/contrib/extprotocol/expected/setup.out @@ -0,0 +1,71 @@ +CREATE SCHEMA exttableext; +GRANT ALL ON SCHEMA exttableext TO PUBLIC; +SET search_path TO 'exttableext'; +-- Create an example table exttabtest that will be used as source table + CREATE TABLE exttabtest( + id int, + name varchar(20), + value1 int, + value2 int + ) + DISTRIBUTED BY (id); +-- Loading 100 records +-- Use only 100 rows for easy to verify the results +-- In order to test multiple data buffers and related edge cases, more data (at least several MBs or more) +-- will be used, as in Performance tests 1M and 100M test cases + \echo 'loading data...' +loading data... + INSERT INTO exttabtest SELECT i, 'name'||i, i*2, i*3 FROM generate_series(1,100) i; +-- Test 1: create read and write functions based on example gpextprotocol.so +-- Note: Only STABLE is supported for protocol, though this has not been enforced at the time of testing + CREATE OR REPLACE FUNCTION write_to_file_stable() RETURNS integer AS + '$libdir/gpextprotocol.so', 'demoprot_export' LANGUAGE C STABLE; + CREATE OR REPLACE FUNCTION read_from_file_stable() RETURNS integer AS + '$libdir/gpextprotocol.so', 'demoprot_import' LANGUAGE C STABLE; + -- Check pg_proc catalog table for new created functions + SELECT proname, prolang,proisstrict,provolatile,pronargs,prorettype,prosrc,proacl FROM pg_proc + WHERE proname like 'write_to_file%' + or proname like 'read_from_file%' + ORDER BY proname; + proname | prolang | proisstrict | provolatile | pronargs | prorettype | prosrc | proacl +-----------------------+---------+-------------+-------------+----------+------------+-----------------+-------- + read_from_file_stable | 13 | f | s | 0 | 23 | demoprot_import | + write_to_file_stable | 13 | f | s | 0 | 23 | demoprot_export | +(2 rows) + +-- Test 2: create bi-directional protocol (read and write) using STABLE functions + DROP PROTOCOL IF EXISTS demoprot; +NOTICE: protocol "demoprot" does not exist, skipping + CREATE PROTOCOL demoprot ( + readfunc = read_from_file_stable, + writefunc = write_to_file_stable + ); + -- Check dependency: pg_depend table + select count(*) from pg_depend + where objid in ( + select oid from pg_extprotocol where ptcname='demoprot'); + count +------- + 2 +(1 row) + + -- Check pg_extprotocol for new created protocol + select extprot.ptcname, proc1.proname readfunc, proc2.proname writefunc + from pg_extprotocol extprot, pg_proc proc1, pg_proc proc2 + where extprot.ptcname='demoprot' + and extprot.ptcreadfn=proc1.oid + and extprot.ptcwritefn=proc2.oid; + ptcname | readfunc | writefunc +----------+-----------------------+---------------------- + demoprot | read_from_file_stable | write_to_file_stable +(1 row) + + DROP EXTERNAL TABLE IF EXISTS clean_exttabtest_files; +NOTICE: table "clean_exttabtest_files" does not exist, skipping + CREATE EXTERNAL WEB TABLE clean_exttabtest_files(stdout text) EXECUTE 'rm -f exttabtest*' ON ALL FORMAT 'text'; + GRANT ALL ON clean_exttabtest_files TO PUBLIC; + SELECT * FROM clean_exttabtest_files; + stdout +-------- +(0 rows) + diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/exttableext.sql b/contrib/extprotocol/sql/exttableext.sql similarity index 94% rename from src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/exttableext.sql rename to contrib/extprotocol/sql/exttableext.sql index 11d10d47e6..f707eae4cc 100644 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/exttableext.sql +++ b/contrib/extprotocol/sql/exttableext.sql @@ -27,9 +27,11 @@ SET search_path TO 'exttableext'; EXCEPT ALL SELECT * FROM exttabtest; - -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from exttabtest_r - GROUP BY 1 ORDER BY 1; +-- verify data should be evenly distributed +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; -- Test 4.1: create uni-directional write protocol -- create WET using created protocol @@ -101,9 +103,11 @@ SET search_path TO 'exttableext'; EXCEPT ALL SELECT * FROM exttabtest; - -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from exttabtest_r_dist - GROUP BY 1 ORDER BY 1; +-- verify data should be evenly distributed +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_dist group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; -- Test 6: using two urls and using CSV format @@ -128,9 +132,11 @@ SET search_path TO 'exttableext'; EXCEPT ALL SELECT * FROM exttabtest; - -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from exttabtest_r_2url - GROUP BY 1 ORDER BY 1; +-- verify data should be evenly distributed +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_2url group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; -- Check the output file at each segments -- ! gpssh -f allsegs ls -l /data/hhuang/MAIN/main_debug/primary/gpseg*/exttabtest_2url*.csv @@ -157,9 +163,11 @@ SET search_path TO 'exttableext'; EXCEPT ALL SELECT * FROM exttabtest; - -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from exttabtest_r_2url - GROUP BY 1 ORDER BY 1; +-- verify data should be evenly distributed +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_2url group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; -- Checking the output files on segments -- ! gpssh -f allsegs ls -l /data/hhuang/MAIN/main_debug/primary/gpseg*/exttabtest_2url*.txt @@ -295,32 +303,34 @@ SET search_path TO 'exttableext'; LOCATION('badprotocol://exttabtest.txt') FORMAT 'text'; --- Test 20: Small dataset - 4 records +-- Test 20: Small dataset - 20 records - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_4records; - CREATE WRITABLE EXTERNAL TABLE exttabtest_w_4records (like exttabtest) - LOCATION('demoprot://exttabtest_4records.txt') + DROP EXTERNAL TABLE IF EXISTS exttabtest_w_20records; + CREATE WRITABLE EXTERNAL TABLE exttabtest_w_20records (like exttabtest) + LOCATION('demoprot://exttabtest_20records.txt') FORMAT 'text' DISTRIBUTED BY (id); - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_4records; - CREATE READABLE EXTERNAL TABLE exttabtest_r_4records (like exttabtest) - LOCATION('demoprot://exttabtest_4records.txt') + DROP EXTERNAL TABLE IF EXISTS exttabtest_r_20records; + CREATE READABLE EXTERNAL TABLE exttabtest_r_20records (like exttabtest) + LOCATION('demoprot://exttabtest_20records.txt') FORMAT 'text'; -- write to WET - INSERT INTO exttabtest_w_4records (SELECT * FROM exttabtest where id<=4); + INSERT INTO exttabtest_w_20records (SELECT * FROM exttabtest where id<=20); -- read from RET - SELECT * FROM exttabtest_r_4records order by id; + SELECT * FROM exttabtest_r_20records order by id; - -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from exttabtest_r_4records - GROUP BY 1 ORDER BY 1; +-- verify data should be evenly distributed +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_20records group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; -- Drop External Tables - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_4records; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_4records; + DROP EXTERNAL TABLE IF EXISTS exttabtest_r_20records; + DROP EXTERNAL TABLE IF EXISTS exttabtest_w_20records; -- Test 21: Small dataset - 1 record DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1record; @@ -597,7 +607,10 @@ DISTRIBUTED BY (id); INSERT INTO formatsource SELECT 'name'||i, i, i*2, i*3 FROM generate_series(1,100) i; -- Check data distribution - SELECT gp_segment_id, count(*) FROM formatsource GROUP BY 1 ORDER BY 1; +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from formatsource group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; -- Test 61: create STABLE read and write functions based on example gpformatter.so -- Note: Only STABLE is supported for formatter. @@ -706,9 +719,12 @@ DROP FUNCTION formatter_import_todrop(); -- Read from RET SELECT count(*) FROM format_r; - -- verify data should be evenly distributed - SELECT gp_segment_id, count(*) from format_r - GROUP BY 1 ORDER BY 1; +-- verify data should be evenly distributed +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from format_r group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; + -- Test 64: Drop format function with external table using the function DROP FUNCTION formatter_export_i(record); @@ -855,16 +871,21 @@ DROP FUNCTION formatter_import_todrop(); insert into format_long select 2.0,i from repeat('oxo2', 1000) i; insert into format_long select 3.0,i from repeat('oxo3', 1000) i; insert into format_long select 4.0,i from repeat('oxo4', 1000) i; + insert into format_long select 5.0,i from repeat('oxo5', 1000) i; + insert into format_long select 6.0,i from repeat('oxo6', 1000) i; + insert into format_long select 7.0,i from repeat('oxo7', 1000) i; + insert into format_long select 8.0,i from repeat('oxo8', 1000) i; + insert into format_long select 9.0,i from repeat('oxo9', 1000) i; -- Check distribution is even - select gp_segment_id,count(*) from format_long - group by gp_segment_id - order by gp_segment_id; +with t as ( + SELECT gp_segment_id as segid, count(*) as cnt from format_long group by gp_segment_id +) +select max(cnt) - min(cnt) > 20 from t; -- Write to WET -- insert should be successful INSERT INTO format_long_w (SELECT * FROM format_long); - -- INSERT 0 4 -- Read from RET select count(*) from format_long_r; @@ -915,15 +936,19 @@ DROP FUNCTION formatter_import_todrop(); insert into format_long select 2.0,i from repeat('oxox2', 1000) i; insert into format_long select 3.0,i from repeat('oxox3', 1000) i; insert into format_long select 4.0,i from repeat('oxox4', 1000) i; + insert into format_long select 5.0,i from repeat('oxox5', 1000) i; + insert into format_long select 6.0,i from repeat('oxox6', 1000) i; + insert into format_long select 7.0,i from repeat('oxox7', 1000) i; + insert into format_long select 8.0,i from repeat('oxox8', 1000) i; + insert into format_long select 9.0,i from repeat('oxox9', 1000) i; -- Write to WET -- insert should be successful INSERT INTO format_long_w (SELECT * FROM format_long); - -- INSERT 0 4 -- Read from RET select count(*) from format_long_r; - -- returns count = 4 + -- returns count = 9 -- Test 73: Interlacing short and long record, testing FMT_NEED_MORE_DATA -- When loading data from RET, long record may trigger FORMATTER_RETURN_NOTIFICATION(fcinfo, FMT_NEED_MORE_DATA). -- Verify the data can be loaded successfully and should exactly match the source records. @@ -960,15 +985,19 @@ DROP FUNCTION formatter_import_todrop(); insert into format_long select 2.0,i from repeat('oxo2', 1000) i; insert into format_long select 3.0,'oxo3'; insert into format_long select 4.0,i from repeat('oxo4', 1000) i; + insert into format_long select 5.0,'oxo5'; + insert into format_long select 6.0,i from repeat('oxo6', 1000) i; + insert into format_long select 7.0,'oxo7'; + insert into format_long select 8.0,i from repeat('oxo8', 1000) i; + insert into format_long select 9.0,'oxo9'; -- Write to WET -- insert should be successful INSERT INTO format_long_w (SELECT * FROM format_long); - -- INSERT 0 4 -- Read from RET select count(*) from format_long_r; - -- returns count = 4 + -- returns count = 9 -- read from RET, both should return 0 SELECT * FROM format_long_r @@ -1091,11 +1120,15 @@ DROP FUNCTION formatter_import_todrop(); insert into formatsource select 2,'oxo2'; insert into formatsource select 3,'oxo3'; insert into formatsource select 4,'oxo4'; + insert into formatsource select 5,'oxo5'; + insert into formatsource select 6,'oxo6'; + insert into formatsource select 7,'oxo7'; + insert into formatsource select 8,'oxo8'; + insert into formatsource select 9,'oxo9'; -- Write to WET -- insert should be successful INSERT INTO format_w (SELECT * FROM formatsource); - -- INSERT 0 4 -- Read from RET select id, value1 from format_r order by id; @@ -1130,11 +1163,15 @@ DROP FUNCTION formatter_import_todrop(); insert into formatsource values (2,null,null); insert into formatsource values (3,null,null); insert into formatsource values (4,null,null); + insert into formatsource values (5,null,null); + insert into formatsource values (6,null,null); + insert into formatsource values (7,null,null); + insert into formatsource values (8,null,null); + insert into formatsource values (9,null,null); -- Write to WET -- insert should be successful INSERT INTO format_w (SELECT * FROM formatsource); - -- INSERT 0 4 -- Read from RET select * from format_r where name is null order by id; diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/setup/query00.sql b/contrib/extprotocol/sql/setup.sql similarity index 51% rename from src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/setup/query00.sql rename to contrib/extprotocol/sql/setup.sql index 115a14bd61..fdaa96a378 100644 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/setup/query00.sql +++ b/contrib/extprotocol/sql/setup.sql @@ -1,53 +1,7 @@ -DROP SCHEMA IF EXISTS exttableext CASCADE; CREATE SCHEMA exttableext; GRANT ALL ON SCHEMA exttableext TO PUBLIC; SET search_path TO 'exttableext'; -- Create an example table exttabtest that will be used as source table - - -- Clean up existing external tables, functions - DROP TABLE IF EXISTS exttabtest; - DROP EXTERNAL TABLE IF EXISTS exttabtest_r; - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1m; - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1m_null; - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_2url; - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_circle; - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_dist; - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_invalid; - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new; - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_null; - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_uni; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1m; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1m_null; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_2url; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_5url; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_circle; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_dist; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_invalid; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_misspath; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_null; - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_uni; - DROP EXTERNAL TABLE IF EXISTS format_long_r; - DROP EXTERNAL TABLE IF EXISTS format_long_w; - DROP EXTERNAL TABLE IF EXISTS format_r; - DROP EXTERNAL TABLE IF EXISTS format_r_s1; - DROP EXTERNAL TABLE IF EXISTS format_r_s2; - DROP EXTERNAL TABLE IF EXISTS format_w; - DROP EXTERNAL TABLE IF EXISTS format_w_s1; - DROP EXTERNAL TABLE IF EXISTS format_w_s2; - - DROP FUNCTION IF EXISTS formatter_export_s(record) CASCADE; - DROP FUNCTION IF EXISTS formatter_export_v(record) CASCADE; - DROP FUNCTION IF EXISTS formatter_import_s() CASCADE; - DROP FUNCTION IF EXISTS formatter_import_v() CASCADE; - DROP FUNCTION IF EXISTS read_from_file() CASCADE; - DROP FUNCTION IF EXISTS read_from_file_stable() CASCADE; - DROP FUNCTION IF EXISTS write_to_file_stable() CASCADE; - DROP FUNCTION IF EXISTS url_validator() CASCADE; - DROP FUNCTION IF EXISTS write_to_file() CASCADE; - DROP FUNCTION IF EXISTS write_to_file_stable() CASCADE; - CREATE TABLE exttabtest( id int, name varchar(20), diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/__init__.py b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/expected/setup/query00.ans b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/expected/setup/query00.ans deleted file mode 100644 index e3050bc3be..0000000000 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/expected/setup/query00.ans +++ /dev/null @@ -1,160 +0,0 @@ -DROP SCHEMA IF EXISTS exttableext; -DROP SCHEMA -CREATE SCHEMA exttableext; -CREATE SCHEMA -GRANT ALL ON SCHEMA exttableext TO PUBLIC; -GRANT -SET search_path TO 'exttablext'; -SET --- Create an example table exttabtest that will be used as source table - -- Clean up existing external tables, functions - DROP TABLE IF EXISTS exttabtest; -psql:/path/sql_file:1: NOTICE: table "exttabtest" does not exist, skipping -DROP TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_r; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1m; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_1m" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1m_null; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_1m_null" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_2url; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_2url" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_circle; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_circle" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_dist; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_dist" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_invalid; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_invalid" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_new" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_null; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_null" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_r_uni; -psql:/path/sql_file:1: NOTICE: table "exttabtest_r_uni" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1m; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_1m" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1m_null; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_1m_null" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_2url; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_2url" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_5url; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_5url" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_circle; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_circle" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_dist; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_dist" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_invalid; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_invalid" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_misspath; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_misspath" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_new" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_null; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_null" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS exttabtest_w_uni; -psql:/path/sql_file:1: NOTICE: table "exttabtest_w_uni" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS format_long_r; -psql:/path/sql_file:1: NOTICE: table "format_long_r" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS format_long_w; -psql:/path/sql_file:1: NOTICE: table "format_long_w" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS format_r; -psql:/path/sql_file:1: NOTICE: table "format_r" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS format_r_s1; -psql:/path/sql_file:1: NOTICE: table "format_r_s1" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS format_r_s2; -psql:/path/sql_file:1: NOTICE: table "format_r_s2" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS format_w; -psql:/path/sql_file:1: NOTICE: table "format_w" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS format_w_s1; -psql:/path/sql_file:1: NOTICE: table "format_w_s1" does not exist, skipping -DROP EXTERNAL TABLE - DROP EXTERNAL TABLE IF EXISTS format_w_s2; -psql:/path/sql_file:1: NOTICE: table "format_w_s2" does not exist, skipping -DROP EXTERNAL TABLE - DROP FUNCTION IF EXISTS formatter_export_s(record) CASCADE; -psql:/path/sql_file:1: NOTICE: function formatter_export_s(record) does not exist, skipping -DROP FUNCTION - DROP FUNCTION IF EXISTS formatter_export_v(record) CASCADE; -psql:/path/sql_file:1: NOTICE: function formatter_export_v(record) does not exist, skipping -DROP FUNCTION - DROP FUNCTION IF EXISTS formatter_import_s() CASCADE; -psql:/path/sql_file:1: NOTICE: function formatter_import_s() does not exist, skipping -DROP FUNCTION - DROP FUNCTION IF EXISTS formatter_import_v() CASCADE; -psql:/path/sql_file:1: NOTICE: function formatter_import_v() does not exist, skipping -DROP FUNCTION - DROP FUNCTION IF EXISTS read_from_file() CASCADE; -psql:/path/sql_file:1: NOTICE: function read_from_file() does not exist, skipping -DROP FUNCTION - DROP FUNCTION IF EXISTS read_from_file_stable() CASCADE; -psql:/path/sql_file:1: NOTICE: function read_from_file_stable() does not exist, skipping -DROP FUNCTION - DROP FUNCTION IF EXISTS write_to_file_stable() CASCADE; -psql:/path/sql_file:1: NOTICE: function write_to_file_stable() does not exist, skipping -DROP FUNCTION - DROP FUNCTION IF EXISTS url_validator() CASCADE; -psql:/path/sql_file:1: NOTICE: function url_validator() does not exist, skipping -DROP FUNCTION - DROP FUNCTION IF EXISTS write_to_file() CASCADE; -psql:/path/sql_file:1: NOTICE: function write_to_file() does not exist, skipping -DROP FUNCTION - DROP FUNCTION IF EXISTS write_to_file_stable() CASCADE; -psql:/path/sql_file:1: NOTICE: function write_to_file_stable() does not exist, skipping -DROP FUNCTION - CREATE TABLE exttabtest( - id int, - name varchar(20), - value1 int, - value2 int - ) - DISTRIBUTED BY (id); -CREATE TABLE --- Loading 100 records --- Use only 100 rows for easy to verify the results --- In order to test multiple data buffers and related edge cases, more data (at least several MBs or more) --- will be used, as in Performance tests 1M and 100M test cases - \echo 'loading data...' -loading data... - INSERT INTO exttabtest SELECT i, 'name'||i, i*2, i*3 FROM generate_series(1,100) i; -INSERT 0 100 - DROP EXTERNAL TABLE IF EXISTS clean_exttabtest_files; -DROP TABLE - CREATE EXTERNAL WEB TABLE clean_exttabtest_files(stdout text) EXECUTE 'rm -f exttabtest*' ON ALL FORMAT 'text'; -CREATE TABLE - GRANT ALL ON clean_exttabtest_files TO PUBLIC; -GRANT - SELECT * FROM clean_exttabtest_files; - stdout --------- -(0 rows) - diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/init_file b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/init_file deleted file mode 100644 index cdc75b4038..0000000000 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/init_file +++ /dev/null @@ -1,5 +0,0 @@ --- start_matchsubs -# Change things like "psql: /Users/mglkey/cdbfast/main/foo/bar.sql:123:" to "PATH" -m/psql:\/.*:\d+:/ -s/psql:\/.*:\d+:/psql:PATH:/ --- end_matchsubs diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/setup/removeOutputFile.sql b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/setup/removeOutputFile.sql deleted file mode 100644 index a220ac7518..0000000000 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/setup/removeOutputFile.sql +++ /dev/null @@ -1,9 +0,0 @@ --- WET using example protocol demoprot will export data files to data directory on primary segments --- If NOT cleaned, these files will break data replication (between primary and mirror). --- Therefore it is necessary to clean up generated data files after each test case run. --- --- query to construct command to remove the generated data file from data directory -SELECT 'gpssh -h '|| gpsc.hostname ||' rm -f '|| fse.fselocation || '/exttabtest*' -FROM pg_filespace_entry fse, gp_segment_configuration gpsc -WHERE gpsc.dbid = fse.fsedbid and gpsc.role = 'p' and gpsc.content > -1 -ORDER BY gpsc.hostname, fse.fselocation; diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/test_exttable.py b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/test_exttable.py deleted file mode 100755 index bdd040845f..0000000000 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/test_exttable.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Copyright (C) 2004-2015 Pivotal Software, Inc. All rights reserved. - -This program and the accompanying materials are made available under -the terms of the under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import os - -from mpp.models import SQLTestCase -from tinctest.lib import local_path, run_shell_command - -class ExternalTableExtensionTests(SQLTestCase): - """ - @tags list_mgmt_expand - @product_version gpdb: [4.3-] - """ - - sql_dir = 'sql/' - ans_dir = 'expected/' - - def tearDown(self): - # execute commands to remove output files created in primary segment's data directories - # This cleans up all the files exported by demoprot on the primary segments. - # expects removeOutputFile.sql in sql/setup to be run and its corredponfing out file available in - # out_dir/setup/removeOutputFile.out - out_file = os.path.join(self.get_out_dir(), 'setup', 'removeOutputFile.out') - with open(out_file) as f: - for line in f: - if (line.find('gpssh ')>=0 and line.find('SELECT')<0): - ok = run_shell_command(line) - if not ok: - raise Exception('Output file remove operation error: %s' %line) - super(ExternalTableExtensionTests, self).tearDown() - -- GitLab