Skip to content

G
Gpdb

Greenplum / Gpdb

通知 7
Star 1
Fork 0
G
Gpdb

体验新版 GitCode，发现更多精彩内容 >>

提交 f8734d9d 编写于 作者: H Heikki Linnakangas
浏览文件
操作

Move tests on 'demoprot' from TINC to contrib, where the source is. 

I had to change some of the tests that used 4 tests rows, to have more (9),
to make them pass on a 3 segment cluster. If you only insert 4 rows to an
external table, some segments might not receive any rows, and reading from
the same file later will fail because the file wasn't created. I also
changed the queries that tested that rows are distributed evenly, to work
with a different number of segments.

Hook up the new tests to installcheck-world.
上级 6133a493
隐藏空白更改
内联 并排
Showing with 398 addition and 861 deletion
GNUmakefile.in
浏览文件 @ f8734d9d
......@@ -121,6 +121,7 @@ installcheck-world:
$(MAKE) -C src/pl installcheck
#$(MAKE) -C src/interfaces/ecpg installcheck
#$(MAKE) -C contrib installcheck
$(MAKE) -C contrib/extprotocol installcheck
$(MAKE) -C gpAux/extensions installcheck
$(MAKE) -C src/bin/gpfdist installcheck
......
contrib/extprotocol/Makefile
浏览文件 @ f8734d9d
MODULE_big = gpextprotocol
OBJS = gpextprotocol.o
REGRESS = setup exttableext
PG_CPPFLAGS = -I$(libpq_srcdir)
PG_LIBS = $(libpq_pgport)
......
src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/expected/exttableext.ans → contrib/extprotocol/expected/exttableext.out
浏览文件 @ f8734d9d
SET search_path TO 'exttableext';
SET
-- Test 3: create RET and WET using created protocol
-- Create external RET and WET
DROP EXTERNAL TABLE IF EXISTS exttabtest_w;
NOTICE: table "exttabtest_w" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w(like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r;
NOTICE: table "exttabtest_r" does not exist, skipping
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE exttabtest_r(like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- Checking pg_exttable for new created RET and WET
select location,fmttype,fmtopts,encoding,writable from pg_exttable
where reloid='exttabtest_r'::regclass
......@@ -34,7 +29,6 @@ CREATE EXTERNAL TABLE
(0 rows)
INSERT INTO exttabtest_w (SELECT * FROM exttabtest);
INSERT 0 100
-- read from RET
SELECT * FROM exttabtest_r
EXCEPT ALL
......@@ -43,54 +37,46 @@ INSERT 0 100
----+------+--------+--------
(0 rows)
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from exttabtest_r
GROUP BY 1 ORDER BY 1;
gp_segment_id | count
---------------+-------
0 | 50
1 | 50
(2 rows)
-- verify data should be evenly distributed
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
?column?
----------
f
(1 row)
-- Test 4.1: create uni-directional write protocol
-- create WET using created protocol
DROP PROTOCOL IF EXISTS demoprot CASCADE;
NOTICE: drop cascades to external table exttabtest_r
NOTICE: drop cascades to external table exttabtest_w
DROP PROTOCOL
CREATE PROTOCOL demoprot (
writefunc = write_to_file_stable
);
CREATE PROTOCOL
-- Create WET with uni-directional protocol
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_uni;
NOTICE: table "exttabtest_w_uni" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_uni(like exttabtest)
LOCATION('demoprot://exttabtest_uni.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
-- write to WET
INSERT INTO exttabtest_w_uni (SELECT * FROM exttabtest);
INSERT 0 100
-- Test 4.2: create uni-directional read protocol
-- create RET using created protocol
DROP PROTOCOL IF EXISTS demoprot CASCADE;
NOTICE: drop cascades to external table exttabtest_w_uni
DROP PROTOCOL
CREATE PROTOCOL demoprot (
readfunc = read_from_file_stable
);
CREATE PROTOCOL
-- Create RET with uni-directional protocol
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_uni;
NOTICE: table "exttabtest_r_uni" does not exist, skipping
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE exttabtest_r_uni(like exttabtest)
LOCATION('demoprot://exttabtest_uni.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- read from RET
SELECT * FROM exttabtest_r_uni
EXCEPT ALL
......@@ -109,30 +95,23 @@ CREATE EXTERNAL TABLE
-- Otherwise "ERROR: demoprot_import: could not open file " will be thrown.
DROP PROTOCOL IF EXISTS demoprot CASCADE;
NOTICE: drop cascades to external table exttabtest_r_uni
DROP PROTOCOL
CREATE PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
CREATE PROTOCOL
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_dist;
NOTICE: table "exttabtest_w_dist" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_dist(like exttabtest)
LOCATION('demoprot://exttabtest_dist.txt')
FORMAT 'text'
DISTRIBUTED BY (value2);
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_dist;
NOTICE: table "exttabtest_r_dist" does not exist, skipping
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE exttabtest_r_dist(like exttabtest)
LOCATION('demoprot://exttabtest_dist.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- write to WET
INSERT INTO exttabtest_w_dist (SELECT * FROM exttabtest);
INSERT 0 100
-- read from RET
SELECT * FROM exttabtest_r_dist
EXCEPT ALL
......@@ -141,36 +120,32 @@ INSERT 0 100
----+------+--------+--------
(0 rows)
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from exttabtest_r_dist
GROUP BY 1 ORDER BY 1;
gp_segment_id | count
---------------+-------
0 | 51
1 | 49
(2 rows)
-- verify data should be evenly distributed
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_dist group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
?column?
----------
f
(1 row)
-- Test 6: using two urls and using CSV format
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_2url;
NOTICE: table "exttabtest_r_2url" does not exist, skipping
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE exttabtest_r_2url (like exttabtest)
LOCATION('demoprot://exttabtest_2url_1.csv',
'demoprot://exttabtest_2url_2.csv')
FORMAT 'csv';
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_2url;
NOTICE: table "exttabtest_w_2url" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_2url (like exttabtest)
LOCATION('demoprot://exttabtest_2url_1.csv',
'demoprot://exttabtest_2url_2.csv')
FORMAT 'csv'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
-- write to WET
INSERT INTO exttabtest_w_2url (SELECT * FROM exttabtest);
INSERT 0 100
-- read from RET
SELECT * FROM exttabtest_r_2url
EXCEPT ALL
......@@ -180,36 +155,32 @@ INSERT 0 100
(0 rows)
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from exttabtest_r_2url
GROUP BY 1 ORDER BY 1;
gp_segment_id | count
---------------+-------
0 | 50
1 | 50
(2 rows)
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_2url group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
?column?
----------
f
(1 row)
-- Check the output file at each segments
-- ! gpssh -f allsegs ls -l /data/hhuang/MAIN/main_debug/primary/gpseg*/exttabtest_2url*.csv
-- Test 7: using two urls and text format
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_2url;
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE exttabtest_r_2url (like exttabtest)
LOCATION('demoprot://exttabtest_2url_1.txt',
'demoprot://exttabtest_2url_2.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_2url;
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_2url (like exttabtest)
LOCATION('demoprot://exttabtest_2url_1.txt',
'demoprot://exttabtest_2url_2.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
-- write to WET
INSERT INTO exttabtest_w_2url (SELECT * FROM exttabtest);
INSERT 0 100
-- read from RET
SELECT * FROM exttabtest_r_2url
EXCEPT ALL
......@@ -219,20 +190,20 @@ INSERT 0 100
(0 rows)
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from exttabtest_r_2url
GROUP BY 1 ORDER BY 1;
gp_segment_id | count
---------------+-------
0 | 50
1 | 50
(2 rows)
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_2url group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
?column?
----------
f
(1 row)
-- Checking the output files on segments
-- ! gpssh -f allsegs ls -l /data/hhuang/MAIN/main_debug/primary/gpseg*/exttabtest_2url*.txt
-- Test 8: Negative - using 5 urls, exceeding number of primary segments (4)
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_5url;
NOTICE: table "exttabtest_w_5url" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_5url (like exttabtest)
LOCATION('demoprot://exttabtest_5url_1.txt',
'demoprot://exttabtest_5url_2.txt',
......@@ -241,10 +212,10 @@ DROP EXTERNAL TABLE
'demoprot://exttabtest_5url_5.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
-- write to WET
INSERT INTO exttabtest_w_5url (SELECT * FROM exttabtest);
psql:/path/sql_file:1: ERROR: External table has more URLs than available primary segments that can write into them (seg0 rh55-qavm55:7532 pid=1235)
ERROR: External table has more URLs than available primary segments that can write into them (seg0 rh55-qavm55:7532 pid=1235)
-- @skip Skipping this test because of a duplicate key violation error.
-- Test 9: Negative - duplicte protocol name (check pg_extprotocol)
SELECT count(*) FROM pg_extprotocol WHERE ptcname='demoprot';
count
......@@ -256,13 +227,13 @@ psql:/path/sql_file:1: ERROR: External table has more URLs than available prima
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
psql:/path/sql_file:1: ERROR: protocol "demoprot" already exists
ERROR: protocol "demoprot" already exists
-- Test 10: Negative: create external table using non-existing protocol
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_neg10 (like exttabtest)
LOCATION('demoprot_nonexist://exttabtest_neg10.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
psql:/path/sql_file:1: ERROR: protocol "demoprot_nonexist" does not exist
ERROR: protocol "demoprot_nonexist" does not exist
-- Test 11: Negative - Using invalid protocol attribute name
-- attribute names must be readproc, write proc, and validatorproc
DROP PROTOCOL IF EXISTS demoprot CASCADE;
......@@ -271,60 +242,53 @@ NOTICE: drop cascades to external table exttabtest_w_2url
NOTICE: drop cascades to external table exttabtest_r_2url
NOTICE: drop cascades to external table exttabtest_r_dist
NOTICE: drop cascades to external table exttabtest_w_dist
DROP PROTOCOL
CREATE PROTOCOL demoprot (
readfunction = read_from_file,
writefunction = write_to_file
);
psql:/path/sql_file:1: ERROR: protocol attribute "readfunction" not recognized
ERROR: protocol attribute "readfunction" not recognized
-- Test 12: Negatvie - using undefined function when defining protocol
DROP PROTOCOL IF EXISTS demoprot CASCADE;
psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping
DROP PROTOCOL
NOTICE: protocol "demoprot" does not exist, skipping
CREATE PROTOCOL demoprot (
readfunc = read_from_file_badname,
writefunc = write_to_file_badname
);
psql:/path/sql_file:1: ERROR: function read_from_file_badname() does not exist
ERROR: function read_from_file_badname() does not exist
-- Test 13: Negatvie - syntax error: missing '=' when defining protocol
DROP PROTOCOL IF EXISTS demoprot CASCADE;
psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping
DROP PROTOCOL
NOTICE: protocol "demoprot" does not exist, skipping
CREATE PROTOCOL demoprot (
readfunc read_from_file,
writefunc write_to_file
);
psql:/path/sql_file:1: ERROR: syntax error at or near "read_from_file"
ERROR: syntax error at or near "read_from_file"
LINE 2: readfunc read_from_file,
^
-- @skip Test disabled in cdbfast as well
-- Test 14: Negative - switching read function and write function
-- This is user error. GPDB should display meaningful error message.
-- Not running this test. Comment from cdbfast - Comment this out since it doesn't make much sense and it creates big output files
DROP PROTOCOL IF EXISTS demoprot CASCADE;
psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping
DROP PROTOCOL
NOTICE: protocol "demoprot" does not exist, skipping
CREATE PROTOCOL demoprot (
writefunc = read_from_file_stable,
readfunc = write_to_file_stable
);
CREATE PROTOCOL
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_switched;
NOTICE: table "exttabtest_r_switched" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_switched (like exttabtest)
LOCATION('demoprot://exttabtest_switched.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_switched;
NOTICE: table "exttabtest_r_switched" does not exist, skipping
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE exttabtest_r_switched(like exttabtest)
LOCATION('demoprot://exttabtest_switched.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- write to WET
INSERT INTO exttabtest_w_switched (SELECT * FROM exttabtest);
psql:/path/sql_file:1: ERROR: demoprot_import: could not open file "exttabtest_switched.txt" for reading: No such file or directory (seg0 rh55-qavm55:7532 pid=32378)
ERROR: demoprot_import: could not open file "exttabtest_switched.txt" for reading: No such file or directory (seg0 rh55-qavm55:7532 pid=32378)
-- read from RET
--SELECT * FROM exttabtest_r_switched
--EXCEPT ALL
......@@ -332,160 +296,149 @@ psql:/path/sql_file:1: ERROR: demoprot_import: could not open file "exttabtest_
-- Test 15: Negative - circular reference
-- write to WET while selecting from RET, and WET and RET are using the same data source files
DROP PROTOCOL IF EXISTS demoprot CASCADE;
psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping
NOTICE: protocol "demoprot" does not exist, skipping
NOTICE: drop cascades to external table exttabtest_r_switched
NOTICE: drop cascades to external table exttabtest_w_switched
DROP PROTOCOL
CREATE PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
CREATE PROTOCOL
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_circle;
NOTICE: table "exttabtest_w_circle" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_circle(like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_circle;
NOTICE: table "exttabtest_r_circle" does not exist, skipping
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE exttabtest_r_circle(like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- First to create exttabtest.txt using WET
INSERT INTO exttabtest_w_circle (SELECT * FROM exttabtest);
INSERT 0 100
-- write to WET while reading from RET,
-- using limit to avoid infinit loop
-- This is to test that using RET and WET inappropriately can get yourself into trouble
INSERT INTO exttabtest_w_circle (SELECT * FROM exttabtest_r_circle order by id limit 100);
INSERT 0 100
-- Test 16: Negative - invalid URL: missing path
drop external table if exists exttabtest_w_misspath;
NOTICE: table "exttabtest_w_misspath" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_misspath(like exttabtest)
LOCATION('demoprot://')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
insert into exttabtest_w_misspath (select * from exttabletest);
psql:/path/sql_file:1: ERROR: relation "exttabletest" does not exist
ERROR: relation "exttabletest" does not exist
LINE 1: insert into exttabtest_w_misspath (select * from exttabletes...
^
-- Test 17: Negative - invalid URL: missing protocol
CREATE READABLE EXTERNAL TABLE exttabtest_r_missprot(like exttabtest)
LOCATION('exttabtest.txt')
FORMAT 'text';
psql:/path/sql_file:1: ERROR: invalid URI 'exttabtest.txt' : undefined structure
ERROR: invalid URI 'exttabtest.txt' : undefined structure
-- Test 18: Negative - invalid URL: invalid path
CREATE READABLE EXTERNAL TABLE exttabtest_r_invalidpath(like exttabtest)
LOCATION('demoprot:\\exttabtest.txt')
FORMAT 'text';
psql:/path/sql_file:1: ERROR: invalid URI 'demoprot:\\exttabtest.txt' : undefined structure
ERROR: invalid URI 'demoprot:\\exttabtest.txt' : undefined structure
-- Test 19: Negative - invalid URL: invalid protocol name
CREATE READABLE EXTERNAL TABLE exttabtest_r_invalidprot(like exttabtest)
LOCATION('badprotocol://exttabtest.txt')
FORMAT 'text';
psql:/path/sql_file:1: ERROR: protocol "badprotocol" does not exist
-- Test 20: Small dataset - 4 records
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_4records;
NOTICE: table "exttabtest_w_4records" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_4records (like exttabtest)
LOCATION('demoprot://exttabtest_4records.txt')
ERROR: protocol "badprotocol" does not exist
-- Test 20: Small dataset - 20 records
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_20records;
NOTICE: table "exttabtest_w_20records" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_20records (like exttabtest)
LOCATION('demoprot://exttabtest_20records.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_4records;
NOTICE: table "exttabtest_r_4records" does not exist, skipping
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE exttabtest_r_4records (like exttabtest)
LOCATION('demoprot://exttabtest_4records.txt')
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_20records;
NOTICE: table "exttabtest_r_20records" does not exist, skipping
CREATE READABLE EXTERNAL TABLE exttabtest_r_20records (like exttabtest)
LOCATION('demoprot://exttabtest_20records.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- write to WET
INSERT INTO exttabtest_w_4records (SELECT * FROM exttabtest where id<=4);
INSERT 0 4
INSERT INTO exttabtest_w_20records (SELECT * FROM exttabtest where id<=20);
-- read from RET
SELECT * FROM exttabtest_r_4records order by id;
SELECT * FROM exttabtest_r_20records order by id;
id | name | value1 | value2
----+-------+--------+--------
----+--------+--------+--------
1 | name1 | 2 | 3
2 | name2 | 4 | 6
3 | name3 | 6 | 9
4 | name4 | 8 | 12
(4 rows)
5 | name5 | 10 | 15
6 | name6 | 12 | 18
7 | name7 | 14 | 21
8 | name8 | 16 | 24
9 | name9 | 18 | 27
10 | name10 | 20 | 30
11 | name11 | 22 | 33
12 | name12 | 24 | 36
13 | name13 | 26 | 39
14 | name14 | 28 | 42
15 | name15 | 30 | 45
16 | name16 | 32 | 48
17 | name17 | 34 | 51
18 | name18 | 36 | 54
19 | name19 | 38 | 57
20 | name20 | 40 | 60
(20 rows)
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from exttabtest_r_4records
GROUP BY 1 ORDER BY 1;
gp_segment_id | count
---------------+-------
0 | 2
1 | 2
(2 rows)
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_20records group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
?column?
----------
f
(1 row)
-- Drop External Tables
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_4records;
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_4records;
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_20records;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_20records;
-- Test 21: Small dataset - 1 record
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1record;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_1record" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_w_1record" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_1record (like exttabtest)
LOCATION('demoprot://exttabtest_1record.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1record;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_1record" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_r_1record" does not exist, skipping
CREATE READABLE EXTERNAL TABLE exttabtest_r_1record (like exttabtest)
LOCATION('demoprot://exttabtest_1record.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- write to WET
INSERT INTO exttabtest_w_1record (SELECT * FROM exttabtest where id = 4);
INSERT 0 1
-- read from RET
-- The implemented example protocol (demoprot) requires data file must be available
-- (even it is empty) for each (primary) segment.
-- This is the limitation of this example (MPP-13811)
-- and will cause following SElECT query to fail
SELECT * FROM exttabtest_r_1record order by id;
psql:/path/sql_file:1: ERROR: demoprot_import: could not open file "exttabtest_1record.txt" for reading: No such file or directory (seg0 slice1 rh55-qavm55:7532 pid=23845)
ERROR: demoprot_import: could not open file "exttabtest_1record.txt" for reading: No such file or directory (seg0 slice1 rh55-qavm55:7532 pid=23845)
DETAIL: External table exttabtest_r_1record, file demoprot://exttabtest_1record.txt
-- Drop External Tables
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1record;
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1record;
DROP EXTERNAL TABLE
-- Test 22: Using /dev/null
-- Using /dev/null for output and input
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_null;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_null" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_w_null" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_null (like exttabtest)
LOCATION('demoprot:///dev/null')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_null;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_null" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_r_null" does not exist, skipping
CREATE READABLE EXTERNAL TABLE exttabtest_r_null (like exttabtest)
LOCATION('demoprot:///dev/null')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- write to WET
INSERT INTO exttabtest_w_null (SELECT * FROM exttabtest);
INSERT 0 100
-- read from RET
SELECT count(id) FROM exttabtest_r_null;
count
......@@ -496,28 +449,21 @@ INSERT 0 100
-- Test 23: Performance - 1M records
-- Load 1M rows of data
TRUNCATE TABLE exttabtest;
TRUNCATE TABLE
INSERT INTO exttabtest SELECT i, 'name'||i, i*2, i*3 FROM generate_series(1,1000000) i;
INSERT 0 1000000
-- Using demoprot, import and export 1M records
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1M;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_1m" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_w_1m" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_1M (like exttabtest)
LOCATION('demoprot://exttabtest_1M.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1M;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_1m" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_r_1m" does not exist, skipping
CREATE READABLE EXTERNAL TABLE exttabtest_r_1M (like exttabtest)
LOCATION('demoprot://exttabtest_1M.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- write to WET
INSERT INTO exttabtest_w_1M (SELECT * FROM exttabtest);
INSERT 0 1000000
-- Time: 1369.028 ms
-- read from RET
SELECT count(id) FROM exttabtest_r_1M;
......@@ -529,32 +475,25 @@ INSERT 0 1000000
-- Time: 869.793 ms
-- Compare to using demoprot protocol and output to /dev/null (no disk IO, thanks to Alan)
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1M_null;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_1m_null" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_w_1m_null" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_1M_null (like exttabtest)
LOCATION('demoprot:///dev/null')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1M_null;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_1m_null" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_r_1m_null" does not exist, skipping
CREATE READABLE EXTERNAL TABLE exttabtest_r_1M_null (like exttabtest)
LOCATION('demoprot:///dev/null')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- write to WET
INSERT INTO exttabtest_w_1M_null (SELECT * FROM exttabtest);
INSERT 0 1000000
-- Time: 1368.173 ms (about same as writing to file - CPU intensive)
-- Test 30: UDF dependency - drop UDF when it does not have dependent protocol
-- Create new UDF that has no dependent
CREATE OR REPLACE FUNCTION write_udf_todrop() RETURNS integer AS
'$libdir/gpextprotocol.so', 'demoprot_export' LANGUAGE C STABLE;
CREATE FUNCTION
CREATE OR REPLACE FUNCTION read_udf_todrop() RETURNS integer AS
'$libdir/gpextprotocol.so', 'demoprot_import' LANGUAGE C STABLE;
CREATE FUNCTION
-- Check pg_proc catalog table for new created functions
SELECT proname, prolang,proisstrict,provolatile,pronargs,prorettype,prosrc,proacl FROM pg_proc
WHERE proname like 'write_udf_todrop'
......@@ -568,9 +507,7 @@ CREATE FUNCTION
-- Drop two UDFs
DROP FUNCTION write_udf_todrop();
DROP FUNCTION
DROP FUNCTION read_udf_todrop();
DROP FUNCTION
-- Check pg_proc catalog table for after drop the UDFs
SELECT proname, prolang,proisstrict,provolatile,pronargs,prorettype,prosrc,proacl FROM pg_proc
WHERE proname like 'write_udf_todrop'
......@@ -627,18 +564,15 @@ NOTICE: drop cascades to external table exttabtest_w_null
NOTICE: drop cascades to external table exttabtest_w_misspath
NOTICE: drop cascades to external table exttabtest_r_circle
NOTICE: drop cascades to external table exttabtest_w_circle
DROP PROTOCOL
CREATE PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
CREATE PROTOCOL
-- CREATE PROTOCOL
-- Drop function cascade.
-- Showing notice message of cascade drop of dependent protocol demoprot
drop function read_from_file_stable() cascade;
psql:/path/sql_file:1: NOTICE: drop cascades to protocol demoprot
DROP FUNCTION
NOTICE: drop cascades to protocol demoprot
-- NOTICE: drop cascades to protocol demoprot
-- DROP FUNCTION
-- Check pg_proc for function read_from_file_stable
......@@ -671,38 +605,31 @@ DROP FUNCTION
-- Recreate function read_from_file_stable
CREATE OR REPLACE FUNCTION read_from_file_stable() RETURNS integer AS
'$libdir/gpextprotocol.so', 'demoprot_import' LANGUAGE C STABLE;
CREATE FUNCTION
-- Restore (recreate) protocol with the same protocol name demoprot
DROP PROTOCOL IF EXISTS demoprot CASCADE;
psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping
DROP PROTOCOL
NOTICE: protocol "demoprot" does not exist, skipping
CREATE PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
CREATE PROTOCOL
-- CREATE PROTOCOL
CREATE READABLE EXTERNAL TABLE exttabtest_r(like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w(like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
-- truncate table exttabtest and load 100 records
TRUNCATE TABLE exttabtest;
TRUNCATE TABLE
INSERT INTO exttabtest SELECT i, 'name'||i, i*2, i*3 FROM generate_series(1,100) i;
INSERT 0 100
-- Check existing WET that using protocol demoprot
SELECT * FROM clean_exttabtest_files;
stdout
--------
(0 rows)
insert into exttabtest_w (select * from exttabtest);
INSERT 0 100
-- Check existing RET that using protocol demoprot
select * from exttabtest_r where id <=4 order by id;
id | name | value1 | value2
......@@ -716,14 +643,10 @@ INSERT 0 100
-- returns 4 records
-- Change ext table name when using custom protocol
ALTER TABLE exttabtest_r rename to exttabtest_r_newname;
ALTER TABLE
-- Check for Add|Drop|Rename column
ALTER EXTERNAL TABLE exttabtest_r_newname ADD COLUMN value3 int;
ALTER EXTERNAL TABLE
ALTER TABLE exttabtest_r_newname RENAME COLUMN value3 to value3_newname;
ALTER TABLE
ALTER EXTERNAL TABLE exttabtest_r_newname DROP COLUMN value3_newname;
ALTER EXTERNAL TABLE
-- Check external table is still accessible after alter operation
select * from exttabtest_r_newname where id <=4 order by id;
id | name | value1 | value2
......@@ -761,17 +684,14 @@ External options: {}
-- Drop external table 'exttabtest_r_newname'
DROP EXTERNAL TABLE exttabtest_r_newname;
DROP EXTERNAL TABLE
-- DROP EXTERNAL TABLE
-- Test 38: Negative - handling invalid data formate when using custom protocol
-- Recreate RET
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_invalid;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_invalid" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_r_invalid" does not exist, skipping
CREATE READABLE EXTERNAL TABLE exttabtest_r_invalid(like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- Using WET to create data source file exttabtest.txt
SELECT * FROM clean_exttabtest_files;
stdout
......@@ -779,7 +699,6 @@ CREATE EXTERNAL TABLE
(0 rows)
INSERT INTO exttabtest_w (select * from exttabtest);
INSERT 0 100
-- Check RET is accessible
SELECT count(*) from exttabtest_r_invalid;
count
......@@ -790,38 +709,32 @@ INSERT 0 100
-- returns count = 100
-- Add a column (value3) to RET
ALTER EXTERNAL TABLE exttabtest_r_invalid ADD COLUMN value3 int;
ALTER EXTERNAL TABLE
-- Access RET again with changed structure, the data file format is invalid now
SELECT count(*) from exttabtest_r_invalid;
psql:/path/sql_file:1: ERROR: missing data for column "value3" (seg0 slice1 rh55-qavm55:7532 pid=14575)
ERROR: missing data for column "value3" (seg0 slice1 rh55-qavm55:7532 pid=14575)
DETAIL: External table exttabtest_r_invalid, line 1 of demoprot://exttabtest.txt: "1 name1 2 3"
-- Drop a column (id) from RET
ALTER EXTERNAL TABLE exttabtest_r_invalid DROP COLUMN id;
ALTER EXTERNAL TABLE
-- Access RET again with changed structure, the data file format is also invalid
-- Comment this out since output file is undeterministic
-- SELECT count(*) from exttabtest_r_invalid where gp_segment_id=0;
-- Recreate WET
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_invalid;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_invalid" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_w_invalid" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_invalid(like exttabtest)
LOCATION('demoprot://exttabtest_invalid.txt')
FORMAT 'text';
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
-- Drop a column (id) from WET
ALTER EXTERNAL TABLE exttabtest_w_invalid DROP COLUMN id;
psql:/path/sql_file:1: NOTICE: Dropping a column that is part of the distribution policy forces a NULL distribution policy
ALTER EXTERNAL TABLE
NOTICE: Dropping a column that is part of the distribution policy forces a NULL distribution policy
-- Write to WET
INSERT INTO exttabtest_w_invalid (SELECT * from exttabtest);
psql:/path/sql_file:1: ERROR: INSERT has more expressions than target columns
ERROR: INSERT has more expressions than target columns
-- Test 60: setup source table
-- This table is our example database table for formatter test
DROP TABLE IF EXISTS formatsource CASCADE;
psql:/path/sql_file:1: NOTICE: table "formatsource" does not exist, skipping
DROP TABLE
NOTICE: table "formatsource" does not exist, skipping
CREATE TABLE formatsource(
name varchar(40),
id float8,
......@@ -829,19 +742,19 @@ CREATE TABLE formatsource(
value2 float8
)
DISTRIBUTED BY (id);
CREATE TABLE
-- Loading 100 records
\echo 'loading data...'
loading data...
INSERT INTO formatsource SELECT 'name'||i, i, i*2, i*3 FROM generate_series(1,100) i;
INSERT 0 100
-- Check data distribution
SELECT gp_segment_id, count(*) FROM formatsource GROUP BY 1 ORDER BY 1;
gp_segment_id | count
---------------+-------
0 | 42
1 | 58
(2 rows)
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from formatsource group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
?column?
----------
f
(1 row)
-- Test 61: create STABLE read and write functions based on example gpformatter.so
-- Note: Only STABLE is supported for formatter.
......@@ -850,27 +763,21 @@ INSERT 0 100
CREATE OR REPLACE FUNCTION formatter_export_s(record) RETURNS bytea
AS '$libdir/gpformatter.so', 'formatter_export'
LANGUAGE C STABLE;
CREATE FUNCTION
CREATE OR REPLACE FUNCTION formatter_import_s() RETURNS record
AS '$libdir/gpformatter.so', 'formatter_import'
LANGUAGE C STABLE;
CREATE FUNCTION
CREATE OR REPLACE FUNCTION formatter_export_v(record) RETURNS bytea
AS '$libdir/gpformatter.so', 'formatter_export'
LANGUAGE C VOLATILE;
CREATE FUNCTION
CREATE OR REPLACE FUNCTION formatter_import_v() RETURNS record
AS '$libdir/gpformatter.so', 'formatter_import'
LANGUAGE C VOLATILE;
CREATE FUNCTION
CREATE OR REPLACE FUNCTION formatter_export_i(record) RETURNS bytea
AS '$libdir/gpformatter.so', 'formatter_export'
LANGUAGE C IMMUTABLE;
CREATE FUNCTION
CREATE OR REPLACE FUNCTION formatter_import_i() RETURNS record
AS '$libdir/gpformatter.so', 'formatter_import'
LANGUAGE C IMMUTABLE;
CREATE FUNCTION
-- Check pg_proc catalog table for new created functions
SELECT proname, prolang,proisstrict,provolatile,pronargs,prorettype,prosrc,proacl FROM pg_proc
WHERE proname like 'formatter%'
......@@ -889,15 +796,11 @@ CREATE FUNCTION
CREATE OR REPLACE FUNCTION formatter_export_todrop(record) RETURNS bytea
AS '$libdir/gpformatter.so', 'formatter_export'
LANGUAGE C STABLE;
CREATE FUNCTION
CREATE OR REPLACE FUNCTION formatter_import_todrop() RETURNS record
AS '$libdir/gpformatter.so', 'formatter_import'
LANGUAGE C STABLE;
CREATE FUNCTION
DROP FUNCTION formatter_export_todrop(record);
DROP FUNCTION
DROP FUNCTION formatter_import_todrop();
DROP FUNCTION
-- Test 63: create RET and WET using demoprot protocol and STABLE formatter
-- Note: Only STABLE is supported for formatter, this is enforced
-- When it is not STABLE (VOLATILE, or IMMUTABLE),
......@@ -906,37 +809,29 @@ DROP FUNCTION
-- Create RET and WET using IMMUTABLE functions will succeed
-- However query such RET or WET should fail
DROP EXTERNAL TABLE IF EXISTS format_w;
psql:/path/sql_file:1: NOTICE: table "format_w" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "format_w" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE format_w(like formatsource)
LOCATION ('demoprot://exttabtest_test63')
FORMAT 'CUSTOM' (FORMATTER='formatter_export_i');
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
DROP EXTERNAL TABLE IF EXISTS format_r;
psql:/path/sql_file:1: NOTICE: table "format_r" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "format_r" does not exist, skipping
CREATE READABLE EXTERNAL TABLE format_r(like formatsource)
LOCATION ('demoprot://exttabtest_test63')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_i');
CREATE EXTERNAL TABLE
INSERT INTO format_w (SELECT * FROM formatsource);
psql:/path/sql_file:1: ERROR: formatter function formatter_export_i is not declared STABLE. (seg1 rh55-qavm55:7533 pid=30568)
ERROR: formatter function formatter_export_i is not declared STABLE. (seg1 rh55-qavm55:7533 pid=30568)
-- ERROR: formatter function formatter_export_i is not declared STABLE. (seg1 rh55-qavm55:7533 pid=14816)
-- Create RET and WET using STABLE functions
DROP EXTERNAL TABLE IF EXISTS format_w;
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE format_w(like formatsource)
LOCATION ('demoprot://exttabtest_test63')
FORMAT 'CUSTOM' (FORMATTER='formatter_export_s');
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
DROP EXTERNAL TABLE IF EXISTS format_r;
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE format_r(like formatsource)
LOCATION ('demoprot://exttabtest_test63')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_s');
CREATE EXTERNAL TABLE
-- Displaying table info
\d format_w
External table "exttableext.format_w"
......@@ -993,7 +888,6 @@ External location: demoprot://exttabtest_test63
(0 rows)
INSERT INTO format_w (SELECT * FROM formatsource);
INSERT 0 100
-- read from RET
SELECT * FROM format_r
EXCEPT ALL
......@@ -1010,23 +904,20 @@ INSERT 0 100
(1 row)
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from format_r
GROUP BY 1 ORDER BY 1;
gp_segment_id | count
---------------+-------
0 | 42
1 | 58
(2 rows)
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from format_r group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
?column?
----------
f
(1 row)
-- Test 64: Drop format function with external table using the function
DROP FUNCTION formatter_export_i(record);
DROP FUNCTION
DROP FUNCTION formatter_import_i();
DROP FUNCTION
DROP FUNCTION formatter_export_s(record);
DROP FUNCTION
DROP FUNCTION formatter_import_s();
DROP FUNCTION
-- Check pg_proc catalog table for the format functions
-- should return 0
SELECT proname, prolang,proisstrict,provolatile,
......@@ -1058,21 +949,19 @@ DROP FUNCTION
(0 rows)
INSERT INTO format_w (SELECT * FROM formatsource);
psql:/path/sql_file:1: ERROR: formatter function formatter_export_s of type writable was not found. (seg0 rh55-qavm55:7532 pid=30616)
ERROR: formatter function formatter_export_s of type writable was not found. (seg0 rh55-qavm55:7532 pid=30616)
-- Read from RET, should fail
SELECT count(*) FROM format_r;
psql:/path/sql_file:1: ERROR: formatter function formatter_import_s of type readable was not found.
ERROR: formatter function formatter_import_s of type readable was not found.
HINT: Create it with CREATE FUNCTION.
-- Test 65: Restore (recreate) functions with same name, external table should work again
-- Recreate UDFs with same function names
CREATE OR REPLACE FUNCTION formatter_export_s(record) RETURNS bytea
AS '$libdir/gpformatter.so', 'formatter_export'
LANGUAGE C STABLE;
CREATE FUNCTION
CREATE OR REPLACE FUNCTION formatter_import_s() RETURNS record
AS '$libdir/gpformatter.so', 'formatter_import'
LANGUAGE C STABLE;
CREATE FUNCTION
-- Check pg_proc catalog table for new created functions
SELECT proname, prolang,proisstrict,provolatile,
pronargs,prorettype,prosrc,proacl FROM pg_proc
......@@ -1091,7 +980,6 @@ CREATE FUNCTION
(0 rows)
INSERT INTO format_w (SELECT * FROM formatsource);
INSERT 0 100
-- Read from RET
SELECT count(*) FROM format_r;
count
......@@ -1105,45 +993,35 @@ INSERT 0 100
CREATE OR REPLACE FUNCTION formatter_export_s(record) RETURNS bytea
AS '$libdir/gpformatter.so', 'formatter_export'
LANGUAGE C STABLE;
CREATE FUNCTION
CREATE OR REPLACE FUNCTION formatter_import_s() RETURNS record
AS '$libdir/gpformatter.so', 'formatter_import'
LANGUAGE C STABLE;
CREATE FUNCTION
-- First pair of RET and WET that using
-- formatter_import_s and formatter_export_s
DROP EXTERNAL TABLE IF EXISTS format_w_s1;
psql:/path/sql_file:1: NOTICE: table "format_w_s1" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "format_w_s1" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE format_w_s1(like formatsource)
LOCATION ('demoprot://exttabtest_test67_s1')
FORMAT 'CUSTOM' (FORMATTER='formatter_export_s');
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
DROP EXTERNAL TABLE IF EXISTS format_r_s1;
psql:/path/sql_file:1: NOTICE: table "format_r_s1" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "format_r_s1" does not exist, skipping
CREATE READABLE EXTERNAL TABLE format_r_s1(like formatsource)
LOCATION ('demoprot://exttabtest_test67_s1')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_s');
CREATE EXTERNAL TABLE
-- Second pair of RET and WET that using
-- same formatter_import_s and formatter_export_s
DROP EXTERNAL TABLE IF EXISTS format_w_s2;
psql:/path/sql_file:1: NOTICE: table "format_w_s2" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "format_w_s2" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE format_w_s2(like formatsource)
LOCATION ('demoprot://exttabtest_test67_s2')
FORMAT 'CUSTOM' (FORMATTER='formatter_export_s');
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
DROP EXTERNAL TABLE IF EXISTS format_r_s2;
psql:/path/sql_file:1: NOTICE: table "format_r_s2" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "format_r_s2" does not exist, skipping
CREATE READABLE EXTERNAL TABLE format_r_s2(like formatsource)
LOCATION ('demoprot://exttabtest_test67_s2')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_s');
CREATE EXTERNAL TABLE
-- Checking pg_exttable
select pg_class.relname, fmttype,
fmtopts, encoding, writable
......@@ -1162,9 +1040,7 @@ CREATE EXTERNAL TABLE
-- Write to WET
INSERT INTO format_w_s1 (SELECT * FROM formatsource);
INSERT 0 100
INSERT INTO format_w_s2 (SELECT * FROM formatsource);
INSERT 0 100
-- read from RET
SELECT * FROM format_r_s1
EXCEPT ALL
......@@ -1196,13 +1072,11 @@ INSERT 0 100
-- Test 71: Check limit of MAX_FORMAT_STRING 4096 bytes for variable length strings - text type
-- Create format_long table for formatter long record test
DROP TABLE IF EXISTS format_long CASCADE;
psql:/path/sql_file:1: NOTICE: table "format_long" does not exist, skipping
DROP TABLE
NOTICE: table "format_long" does not exist, skipping
CREATE TABLE format_long (
id float8,
name text
) DISTRIBUTED by (id);
CREATE TABLE
-- Check the atttypmod of column name is -1
SELECT atttypmod FROM pg_attribute
WHERE attrelid = 'format_long'::regclass
......@@ -1215,80 +1089,67 @@ CREATE TABLE
-- returns atttypmod = -1
-- Create RET and WET using demoprot and custom format UDFs
DROP EXTERNAL TABLE IF EXISTS format_long_w;
psql:/path/sql_file:1: NOTICE: table "format_long_w" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "format_long_w" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE format_long_w(like format_long)
LOCATION ('demoprot://exttabtest_test71.txt')
FORMAT 'CUSTOM' (FORMATTER='formatter_export_s');
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
DROP EXTERNAL TABLE IF EXISTS format_long_r;
psql:/path/sql_file:1: NOTICE: table "format_long_r" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "format_long_r" does not exist, skipping
CREATE READABLE EXTERNAL TABLE format_long_r(like format_long)
LOCATION ('demoprot://exttabtest_test71.txt')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_s');
CREATE EXTERNAL TABLE
-- Loading each record with 4000 characters, less than 4096 bytes
truncate table format_long;
TRUNCATE TABLE
insert into format_long select 1.0,i from repeat('oxo1', 1000) i;
INSERT 0 1
insert into format_long select 2.0,i from repeat('oxo2', 1000) i;
INSERT 0 1
insert into format_long select 3.0,i from repeat('oxo3', 1000) i;
INSERT 0 1
insert into format_long select 4.0,i from repeat('oxo4', 1000) i;
INSERT 0 1
insert into format_long select 5.0,i from repeat('oxo5', 1000) i;
insert into format_long select 6.0,i from repeat('oxo6', 1000) i;
insert into format_long select 7.0,i from repeat('oxo7', 1000) i;
insert into format_long select 8.0,i from repeat('oxo8', 1000) i;
insert into format_long select 9.0,i from repeat('oxo9', 1000) i;
-- Check distribution is even
select gp_segment_id,count(*) from format_long
group by gp_segment_id
order by gp_segment_id;
gp_segment_id | count
---------------+-------
0 | 1
1 | 3
(2 rows)
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from format_long group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
?column?
----------
f
(1 row)
-- Write to WET
-- insert should be successful
INSERT INTO format_long_w (SELECT * FROM format_long);
INSERT 0 4
-- INSERT 0 4
-- Read from RET
select count(*) from format_long_r;
count
-------
4
9
(1 row)
-- returns count = 4
-- Now loading each record with 5000 characters, more than 4096 bytes for each record
truncate table format_long;
TRUNCATE TABLE
insert into format_long select 1.0,i from repeat('oxox1', 1000) i;
INSERT 0 1
insert into format_long select 2.0,i from repeat('oxox2', 1000) i;
INSERT 0 1
insert into format_long select 3.0,i from repeat('oxox3', 1000) i;
INSERT 0 1
insert into format_long select 4.0,i from repeat('oxox4', 1000) i;
INSERT 0 1
-- Write to WET
-- insert should fail since MAX_FORMAT_STRING (4096) is exceeded
INSERT INTO format_long_w (SELECT * FROM format_long);
psql:/path/sql_file:1: ERROR: formatter_export: buffer too small (gpformatter.c:182) (seg0 rh55-qavm55:7532 pid=22752) (cdbdisp.c:1453)
ERROR: formatter_export: buffer too small (gpformatter.c:182) (seg0 rh55-qavm55:7532 pid=22752) (cdbdisp.c:1453)
-- ERROR: formatter_export: buffer too small (gpformatter.c:182) (seg2 rh55-qavm58:5532 pid=20093)
-- Test 72: Verify limit of MAX_FORMAT_STRING 4096 bytes does not apply to fixed length strings - varchar()
-- MAX_FORMAT_STRING limit should not apply to fixed length strings (char or varchar)
-- Create format_long table for formatter long record test
DROP TABLE IF EXISTS format_long CASCADE;
DROP TABLE
CREATE TABLE format_long (
id float8,
name varchar(6000)
) DISTRIBUTED by (id);
CREATE TABLE
-- Check the atttypmod of name > 0
SELECT atttypmod FROM pg_attribute
WHERE attrelid = 'format_long'::regclass
......@@ -1301,53 +1162,45 @@ CREATE TABLE
-- returns atttypmod = 6004
-- Create RET and WET using demoprot and custom format UDFs
DROP EXTERNAL TABLE IF EXISTS format_long_w;
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE format_long_w(like format_long)
LOCATION ('demoprot://exttabtest_test72.txt')
FORMAT 'CUSTOM' (FORMATTER='formatter_export_s');
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
DROP EXTERNAL TABLE IF EXISTS format_long_r;
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE format_long_r(like format_long)
LOCATION ('demoprot://exttabtest_test72.txt')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_s');
CREATE EXTERNAL TABLE
-- Now loading each record with 5000 characters, more than 4096 bytes for each record
truncate table format_long;
TRUNCATE TABLE
insert into format_long select 1.0,i from repeat('oxox1', 1000) i;
INSERT 0 1
insert into format_long select 2.0,i from repeat('oxox2', 1000) i;
INSERT 0 1
insert into format_long select 3.0,i from repeat('oxox3', 1000) i;
INSERT 0 1
insert into format_long select 4.0,i from repeat('oxox4', 1000) i;
INSERT 0 1
insert into format_long select 5.0,i from repeat('oxox5', 1000) i;
insert into format_long select 6.0,i from repeat('oxox6', 1000) i;
insert into format_long select 7.0,i from repeat('oxox7', 1000) i;
insert into format_long select 8.0,i from repeat('oxox8', 1000) i;
insert into format_long select 9.0,i from repeat('oxox9', 1000) i;
-- Write to WET
-- insert should be successful
INSERT INTO format_long_w (SELECT * FROM format_long);
INSERT 0 4
-- INSERT 0 4
-- Read from RET
select count(*) from format_long_r;
count
-------
4
9
(1 row)
-- returns count = 4
-- returns count = 9
-- Test 73: Interlacing short and long record, testing FMT_NEED_MORE_DATA
-- When loading data from RET, long record may trigger FORMATTER_RETURN_NOTIFICATION(fcinfo, FMT_NEED_MORE_DATA).
-- Verify the data can be loaded successfully and should exactly match the source records.
-- Create format_long table for formatter long record test
DROP TABLE IF EXISTS format_long CASCADE;
DROP TABLE
CREATE TABLE format_long (
id float8,
name varchar(5000)
) DISTRIBUTED by (id);
CREATE TABLE
-- Check the atttypmod of name > 0
SELECT atttypmod FROM pg_attribute
WHERE attrelid = 'format_long'::regclass
......@@ -1360,44 +1213,38 @@ CREATE TABLE
-- returns atttypmod = 5004
-- Create RET and WET using demoprot and custom format UDFs
DROP EXTERNAL TABLE IF EXISTS format_long_w;
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE format_long_w(like format_long)
LOCATION ('demoprot://exttabtest_test73.txt')
FORMAT 'CUSTOM' (FORMATTER='formatter_export_s');
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
DROP EXTERNAL TABLE IF EXISTS format_long_r;
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE format_long_r(like format_long)
LOCATION ('demoprot://exttabtest_test73.txt')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_s');
CREATE EXTERNAL TABLE
-- Loading short and long records alternatively.
-- This should trigger FORMATTER_RETURN_NOTIFICATION(fcinfo, FMT_NEED_MORE_DATA)
-- However not really sure about this.
truncate table format_long;
TRUNCATE TABLE
insert into format_long select 1.0,'oxo1';
INSERT 0 1
insert into format_long select 2.0,i from repeat('oxo2', 1000) i;
INSERT 0 1
insert into format_long select 3.0,'oxo3';
INSERT 0 1
insert into format_long select 4.0,i from repeat('oxo4', 1000) i;
INSERT 0 1
insert into format_long select 5.0,'oxo5';
insert into format_long select 6.0,i from repeat('oxo6', 1000) i;
insert into format_long select 7.0,'oxo7';
insert into format_long select 8.0,i from repeat('oxo8', 1000) i;
insert into format_long select 9.0,'oxo9';
-- Write to WET
-- insert should be successful
INSERT INTO format_long_w (SELECT * FROM format_long);
INSERT 0 4
-- INSERT 0 4
-- Read from RET
select count(*) from format_long_r;
count
-------
4
9
(1 row)
-- returns count = 4
-- returns count = 9
-- read from RET, both should return 0
SELECT * FROM format_long_r
EXCEPT ALL
......@@ -1410,47 +1257,36 @@ INSERT 0 4
-- SAS example formatter only support String and Float types. For other types of data (like INT), it throws error "unsupported data type"
-- Create format_long table for formatter long record test
DROP TABLE IF EXISTS format_long CASCADE;
DROP TABLE
CREATE TABLE format_long (
id int,
name text
) DISTRIBUTED by (id);
CREATE TABLE
-- Create RET and WET using demoprot and custom format UDFs
DROP EXTERNAL TABLE IF EXISTS format_long_w;
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE format_long_w(like format_long)
LOCATION ('demoprot://format_long_test14')
FORMAT 'CUSTOM' (FORMATTER='formatter_export_s');
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
DROP EXTERNAL TABLE IF EXISTS format_long_r;
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE format_long_r(like format_long)
-- using source data file format_long_test13 created by previous test
LOCATION ('demoprot://format_long_test13')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_s');
CREATE EXTERNAL TABLE
-- Now loading each record with 4000 characters, less than 4096 bytes for each record
truncate table format_long;
TRUNCATE TABLE
insert into format_long select 1,'oxo1';
INSERT 0 1
insert into format_long select 2,'oxo2';
INSERT 0 1
insert into format_long select 3,'oxo3';
INSERT 0 1
insert into format_long select 4,'oxo4';
INSERT 0 1
-- Write to WET
-- insert should fail since INT type is not supported
INSERT INTO format_long_w (SELECT * FROM format_long);
psql:/path/sql_file:1: ERROR: formatter_export error: unsupported data type (gpformatter.c:100) (seg1 rh55-qavm55:7533 pid=27134) (cdbdisp.c:1453)
ERROR: formatter_export error: unsupported data type (gpformatter.c:100) (seg1 rh55-qavm55:7533 pid=27134) (cdbdisp.c:1453)
-- ERROR: formatter_export error: unsupported data type (gpformatter.c:100) (seg2 rh55-qavm58:5532 pid=20668) (cdbdisp.c:1458)
-- Read from RET using data file format_long_test13 create by previous test
-- select should fail since INT type is not supported
select count(*) from format_long_r;
psql:/path/sql_file:1: ERROR: demoprot_import: could not open file "format_long_test13" for reading: No such file or directory (seg1 slice1 rh55-qavm55:7533 pid=27134)
ERROR: demoprot_import: could not open file "format_long_test13" for reading: No such file or directory (seg1 slice1 rh55-qavm55:7533 pid=27134)
DETAIL: External table format_long_r
-- ERROR: formatter_import error: unsupported data type (gpformatter.c:256) (seg1 slice1 rh55-qavm57:5533 pid=20204) (cdbdisp.c:1458)
-- Test 75: External table contains dropped columns
......@@ -1458,55 +1294,41 @@ DETAIL: External table format_long_r
-- Both import and export operations will fail. This is expected.
-- Create example source table formatsource
DROP TABLE IF EXISTS formatsource CASCADE;
DROP TABLE
CREATE TABLE formatsource (
id float8,
name text,
value1 float8
) DISTRIBUTED by (id);
CREATE TABLE
-- Create RET and WET using demoprot and custom format UDFs
DROP EXTERNAL TABLE IF EXISTS format_w;
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE format_w(like formatsource)
LOCATION ('demoprot://format_test15')
FORMAT 'CUSTOM' (FORMATTER='formatter_export_s');
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
DROP EXTERNAL TABLE IF EXISTS format_r;
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE format_r(like formatsource)
-- using source data file format_long_test13 created by previous test
LOCATION ('demoprot://format_long_test13')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_s');
CREATE EXTERNAL TABLE
-- Drop colum value1 from source table formatsource
ALTER TABLE formatsource DROP COLUMN value1;
ALTER TABLE
-- Drop colum value1 from RET and WET
ALTER EXTERNAL TABLE format_r DROP COLUMN value1;
ALTER EXTERNAL TABLE
ALTER EXTERNAL TABLE format_w DROP COLUMN value1;
ALTER EXTERNAL TABLE
-- Loading records
truncate table formatsource;
TRUNCATE TABLE
insert into formatsource select 1,'oxo1';
INSERT 0 1
insert into formatsource select 2,'oxo2';
INSERT 0 1
insert into formatsource select 3,'oxo3';
INSERT 0 1
insert into formatsource select 4,'oxo4';
INSERT 0 1
-- Write to WET
-- insert should failed because of dropped column value1
INSERT INTO format_w (SELECT * FROM formatsource);
psql:/path/sql_file:1: ERROR: formatter_export: dropped columns (gpformatter.c:80) (seg0 rh55-qavm55:7532 pid=27212) (cdbdisp.c:1453)
ERROR: formatter_export: dropped columns (gpformatter.c:80) (seg0 rh55-qavm55:7532 pid=27212) (cdbdisp.c:1453)
-- ERROR: formatter_export: dropped columns (gpformatter.c:80) (seg1 rh55-qavm57:5533 pid=20454) (cdbdisp.c:1458)
-- Read from RET using data file format_long_test13 create by previous test
select count(*) from format_r;
psql:/path/sql_file:1: ERROR: demoprot_import: could not open file "format_long_test13" for reading: No such file or directory (seg0 slice1 rh55-qavm55:7532 pid=27212)
ERROR: demoprot_import: could not open file "format_long_test13" for reading: No such file or directory (seg0 slice1 rh55-qavm55:7532 pid=27212)
DETAIL: External table format_r
-- ERROR: formatter_import: dropped columns (gpformatter.c:244) (seg2 slice1 rh55-qavm58:5532 pid=20911) (cdbdisp.c:1458)
-- Test 76: External table contains added column
......@@ -1514,47 +1336,37 @@ DETAIL: External table format_r
-- The value of the added column would be either null or NaN, depends on the data type.
-- Create example source table formatsource
DROP TABLE IF EXISTS formatsource CASCADE;
DROP TABLE
CREATE TABLE formatsource (
id float8,
name char(10)
) DISTRIBUTED by (id);
CREATE TABLE
-- Create RET and WET using demoprot and custom format UDFs
DROP EXTERNAL TABLE IF EXISTS format_w;
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE format_w(like formatsource)
LOCATION ('demoprot://exttabtest_test76.txt')
FORMAT 'CUSTOM' (FORMATTER='formatter_export_s');
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
DROP EXTERNAL TABLE IF EXISTS format_r;
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE format_r(like formatsource)
LOCATION ('demoprot://exttabtest_test76.txt')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_s');
CREATE EXTERNAL TABLE
-- Add column value1 to RET and WET
ALTER EXTERNAL TABLE format_r ADD COLUMN value1 float8;
ALTER EXTERNAL TABLE
ALTER EXTERNAL TABLE format_w ADD COLUMN value1 float8;
ALTER EXTERNAL TABLE
-- Loading records
truncate table formatsource;
TRUNCATE TABLE
insert into formatsource select 1,'oxo1';
INSERT 0 1
insert into formatsource select 2,'oxo2';
INSERT 0 1
insert into formatsource select 3,'oxo3';
INSERT 0 1
insert into formatsource select 4,'oxo4';
INSERT 0 1
insert into formatsource select 5,'oxo5';
insert into formatsource select 6,'oxo6';
insert into formatsource select 7,'oxo7';
insert into formatsource select 8,'oxo8';
insert into formatsource select 9,'oxo9';
-- Write to WET
-- insert should be successful
INSERT INTO format_w (SELECT * FROM formatsource);
INSERT 0 4
-- INSERT 0 4
-- Read from RET
select id, value1 from format_r order by id;
id | value1
......@@ -1563,7 +1375,12 @@ INSERT 0 4
2 | NaN
3 | NaN
4 | NaN
(4 rows)
5 | NaN
6 | NaN
7 | NaN
8 | NaN
9 | NaN
(9 rows)
-- Test 77: data with null values
-- SAS example formatter casts null value to:
......@@ -1571,43 +1388,35 @@ INSERT 0 4
-- * NaN for Float8
-- Create example source table formatsource
DROP TABLE IF EXISTS formatsource CASCADE;
DROP TABLE
CREATE TABLE formatsource (
id float8,
name text,
value1 float8
) DISTRIBUTED by (id);
CREATE TABLE
-- Create RET and WET using demoprot and custom format UDFs
DROP EXTERNAL TABLE IF EXISTS format_w;
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE format_w(like formatsource)
LOCATION ('demoprot://exttabtest_test77.txt')
FORMAT 'CUSTOM' (FORMATTER='formatter_export_s');
psql:/path/sql_file:1: NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
CREATE EXTERNAL TABLE
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
DROP EXTERNAL TABLE IF EXISTS format_r;
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE format_r(like formatsource)
LOCATION ('demoprot://exttabtest_test77.txt')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_s');
CREATE EXTERNAL TABLE
-- Loading records
truncate table formatsource;
TRUNCATE TABLE
insert into formatsource values (1,null,null);
INSERT 0 1
insert into formatsource values (2,null,null);
INSERT 0 1
insert into formatsource values (3,null,null);
INSERT 0 1
insert into formatsource values (4,null,null);
INSERT 0 1
insert into formatsource values (5,null,null);
insert into formatsource values (6,null,null);
insert into formatsource values (7,null,null);
insert into formatsource values (8,null,null);
insert into formatsource values (9,null,null);
-- Write to WET
-- insert should be successful
INSERT INTO format_w (SELECT * FROM formatsource);
INSERT 0 4
-- INSERT 0 4
-- Read from RET
select * from format_r where name is null order by id;
id | name | value1
......@@ -1616,26 +1425,27 @@ INSERT 0 4
2 | | NaN
3 | | NaN
4 | | NaN
(4 rows)
5 | | NaN
6 | | NaN
7 | | NaN
8 | | NaN
9 | | NaN
(9 rows)
-- Test 78: Read from empty data file
-- SAS example formatter can correctly handle empty input, either from /dev/null or from empty input data files.
-- Create example source table formatsource
DROP TABLE IF EXISTS formatsource CASCADE;
DROP TABLE
CREATE TABLE formatsource (
id float8,
name text,
value1 float8
) DISTRIBUTED by (id);
CREATE TABLE
-- Create RET with custom format using /dev/null as source file
DROP EXTERNAL TABLE IF EXISTS format_r;
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE format_r(like formatsource)
LOCATION ('demoprot:///dev/null')
FORMAT 'CUSTOM' (FORMATTER='formatter_import_s');
CREATE EXTERNAL TABLE
-- Read from RET using /dev/null
select * from format_r;
id | name | value1
......@@ -1646,14 +1456,11 @@ CREATE EXTERNAL TABLE
-- create the protocol read and write STABLE functions
CREATE OR REPLACE FUNCTION write_to_file() RETURNS integer AS
'$libdir/gpextprotocol.so', 'demoprot_export' LANGUAGE C STABLE;
CREATE FUNCTION
CREATE OR REPLACE FUNCTION read_from_file() RETURNS integer AS
'$libdir/gpextprotocol.so', 'demoprot_import' LANGUAGE C STABLE;
CREATE FUNCTION
-- create validation STABLE function
CREATE OR REPLACE FUNCTION url_validator() RETURNS void AS
'$libdir/gpextprotocol.so', 'demoprot_validate_urls' LANGUAGE C STABLE;
CREATE FUNCTION
-- declare the protocol name along with in/out funcs and validator func
DROP PROTOCOL IF EXISTS demoprot CASCADE;
NOTICE: drop cascades to external table format_r
......@@ -1666,13 +1473,12 @@ NOTICE: drop cascades to external table format_r_s1
NOTICE: drop cascades to external table format_w_s1
NOTICE: drop cascades to external table exttabtest_w_invalid
NOTICE: drop cascades to external table exttabtest_r_invalid
DROP PROTOCOL
NOTICE: drop cascades to external table exttabtest_w
CREATE PROTOCOL demoprot (
readfunc = read_from_file,
writefunc = write_to_file,
validatorfunc = url_validator
);
CREATE PROTOCOL
-- Test 82: At ext table create time, validate number of URLs cannot exceed number of primary segments
CREATE READABLE EXTERNAL TABLE exttabtest_5url_r(like exttabtest)
LOCATION('demoprot://test1.txt',
......@@ -1681,96 +1487,79 @@ CREATE PROTOCOL
'demoprot://test4.txt',
'demoprot://test5.txt')
FORMAT 'text';
psql:/path/sql_file:1: ERROR: more than 2 urls aren't allowed in this protocol
ERROR: more than 2 urls aren't allowed in this protocol
-- ERROR: more than 2 urls aren't allowed in this protocol
-- Test 83: At ext table create time, url string cannot contain "secured_directory"
CREATE READABLE EXTERNAL TABLE exttabtest_3url_r(like exttabtest)
LOCATION('demoprot://test1.txt',
'demoprot://secured_directory/test2.txt')
FORMAT 'text';
psql:/path/sql_file:1: ERROR: using 'secured_directory' in a url isn't allowed
ERROR: using 'secured_directory' in a url isn't allowed
-- ERROR: using 'secured_directory' in a url isn't allowed
-- Test 4: Negative - validator protocol function must return void
-- create the validator function and returns integer, which is invalid
DROP FUNCTION IF EXISTS url_validator();
DROP FUNCTION
CREATE OR REPLACE FUNCTION url_validator() RETURNS integer AS
'$libdir/gpextprotocol.so', 'demoprot_validate_urls' LANGUAGE C STABLE;
CREATE FUNCTION
-- declare the protocol name along with in/out funcs and validator func
DROP PROTOCOL IF EXISTS demoprot CASCADE;
DROP PROTOCOL
CREATE PROTOCOL demoprot (
readfunc = read_from_file,
writefunc = write_to_file,
validatorfunc = url_validator
);
psql:/path/sql_file:1: ERROR: validator protocol function url_validator() must return void
ERROR: validator protocol function url_validator() must return void
-- Test 5: Negative - invalid protocol attribute name for validator function: must be "validatorfunc"
-- declare the protocol using invalid attribute name "validatorproc"
DROP PROTOCOL IF EXISTS demoprot CASCADE;
psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping
DROP PROTOCOL
NOTICE: protocol "demoprot" does not exist, skipping
CREATE PROTOCOL demoprot (
readfunc = read_from_file,
writefunc = write_to_file,
validatorproc = url_validator
);
psql:/path/sql_file:1: ERROR: protocol attribute "validatorproc" not recognized
ERROR: protocol attribute "validatorproc" not recognized
-- ERROR: protocol attribute "validatorproc" not recognized
-- ERROR: using 'secured_directory' in a url isn't allowed
-- Create multiple roles with login option so that they can be used for protocol permission tests and alter protocol tests
-- Create another suerpuer user demoprot_super
drop role if exists demoprot_super;
DROP ROLE
create role demoprot_super with SUPERUSER LOGIN;
CREATE ROLE
-- Create a non-privileged user demoprot_nopriv
drop role if exists demoprot_nopriv;
DROP ROLE
create role demoprot_nopriv with login ;
NOTICE: resource queue required -- using default resource queue "pg_default"
CREATE ROLE
-- Create a gphdfs_user with CREATEEXTTABLE privilege using gphdfs protocol
drop role if exists gphdfs_user;
DROP ROLE
create role gphdfs_user with login CREATEEXTTABLE (protocol='gphdfs');
psql:/path/sql_file:1: WARNING: GRANT/REVOKE on gphdfs is deprecated
WARNING: GRANT/REVOKE on gphdfs is deprecated
HINT: Issue the GRANT or REVOKE on the protocol itself
psql:/path/sql_file:1: NOTICE: resource queue required -- using default resource queue "pg_default"
CREATE ROLE
NOTICE: resource queue required -- using default resource queue "pg_default"
-- WARNING: GRANT/REVOKE on gphdfs is deprecated
-- HINT: Issue the GRANT or REVOKE on the protocol itself
-- NOTICE: resource queue required -- using default resource queue "pg_default"
-- CREATE ROLE
-- Test 92: Rename existing protocol
DROP FUNCTION IF EXISTS url_validator();
DROP FUNCTION
CREATE OR REPLACE FUNCTION url_validator() RETURNS void AS
'$libdir/gpextprotocol.so', 'demoprot_validate_urls' LANGUAGE C STABLE;
CREATE FUNCTION
CREATE PROTOCOL demoprot (
readfunc = read_from_file,
writefunc = write_to_file,
validatorfunc = url_validator
);
CREATE PROTOCOL
-- Create external RET and WET
DROP EXTERNAL TABLE IF EXISTS exttabtest_w;
NOTICE: table "exttabtest_w" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w(like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_r" does not exist, skipping
CREATE READABLE EXTERNAL TABLE exttabtest_r(like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- write to WET
SELECT * FROM clean_exttabtest_files;
stdout
......@@ -1778,10 +1567,8 @@ CREATE EXTERNAL TABLE
(0 rows)
INSERT INTO exttabtest_w (SELECT * FROM exttabtest);
INSERT 0 100
-- Rename existing protocol
ALTER PROTOCOL demoprot RENAME to demoprot_new;
ALTER PROTOCOL
-- checking pg_extprotocol
select ptcname, ptctrusted
from pg_extprotocol
......@@ -1809,34 +1596,29 @@ External location: demoprot://exttabtest.txt
-- Create a new ext table using the new protocol name
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_new" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_w_new" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new(like exttabtest)
LOCATION('demoprot_new://exttabtest.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_new" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_r_new" does not exist, skipping
CREATE READABLE EXTERNAL TABLE exttabtest_r_new(like exttabtest)
LOCATION('demoprot_new://exttabtest.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- Verify access old ext table that referencing old protocol name would fail
-- This is expected.
select * from exttabtest_r;
psql:/path/sql_file:1: ERROR: protocol "demoprot" does not exist (seg0 slice1 rh55-qavm55:7532 pid=18394)
ERROR: protocol "demoprot" does not exist (seg0 slice1 rh55-qavm55:7532 pid=18394)
-- ERROR: protocol "demoprot" does not exist (seg1 slice1 rh55-qavm57:5533 pid=8558)
-- Verify access new ext table would be successful
-- However demoprot implementation prevents using any other protocol name than "demoprot"
-- therefore the error is expected.
select count(*) from exttabtest_r_new;
psql:/path/sql_file:1: ERROR: internal error: demoprot called with a different protocol (demoprot_new) (gpextprotocol.c:86) (seg0 slice1 rh55-qavm55:7532 pid=18394) (cdbdisp.c:1453)
ERROR: internal error: demoprot called with a different protocol (demoprot_new) (gpextprotocol.c:87) (seg0 slice1 rh55-qavm55:7532 pid=18394) (cdbdisp.c:1453)
DETAIL: External table exttabtest_r_new, file demoprot_new://exttabtest.txt
-- Rename protocol name back to demoprot
ALTER PROTOCOL demoprot_new RENAME to demoprot;
ALTER PROTOCOL
-- Verify access old ext table that referencing old protocol name would succeed
select count(*) from exttabtest_r;
count
......@@ -1854,12 +1636,10 @@ NOTICE: drop cascades to external table exttabtest_r_new
NOTICE: drop cascades to external table exttabtest_w_new
NOTICE: drop cascades to external table exttabtest_r
NOTICE: drop cascades to external table exttabtest_w
DROP PROTOCOL
CREATE TRUSTED PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
CREATE PROTOCOL
-- Check the owner of trusted protocl demoprot is current user
select ptcname, ptctrusted
from pg_extprotocol join pg_user
......@@ -1873,14 +1653,11 @@ CREATE PROTOCOL
-- Change protocol demoprot owner to non-privileged user "demoprot_nopriv"
ALTER PROTOCOL demoprot OWNER TO demoprot_nopriv;
ALTER PROTOCOL
-- Drop the existing external RET and WET
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new;
NOTICE: table "exttabtest_r_new" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new;
NOTICE: table "exttabtest_w_new" does not exist, skipping
DROP EXTERNAL TABLE
-- Check the owner of demoprot is demoprot_nopriv
-- and no protocol permission has been granted
select ptcname, ptctrusted,ptcacl, usename
......@@ -1908,14 +1685,11 @@ DROP EXTERNAL TABLE
-- to non-privileged user "demoprot_nopriv" so that this user
-- can try to create external table using format: (like exttabtest)
GRANT SELECT ON exttabtest TO demoprot_nopriv;
GRANT
-- As superuser, REVOKE ALL privileges on protocol from owner demoprot_nopriv
REVOKE ALL ON PROTOCOL demoprot FROM demoprot_nopriv;
REVOKE
-- connect as non-privileged user "demoprot_nopriv"
-- which is the owner of trusted demoprot
SET ROLE demoprot_nopriv;
SET
select user;
current_user
-----------------
......@@ -1930,12 +1704,10 @@ SET
CREATE READABLE EXTERNAL TABLE exttabtest_r_new (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new(like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
-- Verify non-privileged user "demoprot_nopriv" can export data via new created WET exttabtest_w_new
SELECT * FROM clean_exttabtest_files;
stdout
......@@ -1943,7 +1715,6 @@ CREATE EXTERNAL TABLE
(0 rows)
INSERT INTO exttabtest_w_new (SELECT * FROM exttabtest);
INSERT 0 100
-- Verify non-privileged user "demoprot_nopriv" can load data via new created RET exttabtest_r_new
select count(*) from exttabtest_r_new;
count
......@@ -1955,41 +1726,34 @@ INSERT 0 100
DROP PROTOCOL demoprot CASCADE;
NOTICE: drop cascades to external table exttabtest_w_new
NOTICE: drop cascades to external table exttabtest_r_new
DROP PROTOCOL
RESET ROLE;
RESET
-- Test 94: Untrusted protocol - Change ownership
-- The owner of untrusted protocol (not a superuser) can still create external table
-- using the untrusted protocol.
-- connect as superuser
-- connect as superuser
-- create untrusted protocol demoprot
DROP PROTOCOL IF EXISTS demoprot CASCADE;
psql:/path/sql_file:1: NOTICE: protocol "demoprot" does not exist, skipping
DROP PROTOCOL
CREATE PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
CREATE PROTOCOL
-- Try to change protocol demoprot owner to non-privileged user "demoprot_nopriv"
-- Should get: ERROR: untrusted protocol "demoprot" can't be owned by non superuser
-- Therefore the owner of trusted protocl demoprot is still the current superuser
ALTER PROTOCOL demoprot OWNER TO demoprot_nopriv;
psql:/path/sql_file:1: ERROR: untrusted protocol "demoprot" can't be owned by non superuser
ERROR: untrusted protocol "demoprot" can't be owned by non superuser
-- As superuser, GRANT SELECT permission on heap table "exttabtest"
-- to non-privileged user "demoprot_nopriv" so that this user
-- can try to create external table using format: (like exttabtest)
GRANT SELECT ON exttabtest TO demoprot_nopriv;
GRANT
-- As superuser, REVOKE ALL privileges on protocol from owner demoprot_nopriv
-- The error is correctly shown
REVOKE ALL ON PROTOCOL demoprot FROM demoprot_nopriv;
psql:/path/sql_file:1: ERROR: protocol "demoprot" is not trusted
ERROR: protocol "demoprot" is not trusted
HINT: Only superusers may use untrusted protocols.
-- ERROR: protocol "demoprot" is not trusted
-- login as non-privileged user "demoprot_nopriv"
SET ROLE demoprot_nopriv;
SET
select user;
current_user
-----------------
......@@ -2003,47 +1767,44 @@ SET
CREATE READABLE EXTERNAL TABLE exttabtest_r_new2 (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text';
psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot
ERROR: permission denied for external protocol demoprot
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new2 (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot
ERROR: permission denied for external protocol demoprot
-- Verified non superuser cannot drop the protocol
DROP PROTOCOL demoprot;
psql:/path/sql_file:1: ERROR: must be owner of external protocol demoprot
ERROR: must be owner of external protocol demoprot
-- Test 95: Alter protocol negative tests
ALTER PROTOCOL demoprot (
readfunc = read_from_file_immutable,
writefunc = write_to_file_immutable
);
psql:/path/sql_file:1: ERROR: syntax error at or near "("
ERROR: syntax error at or near "("
LINE 1: ALTER PROTOCOL demoprot (
^
-- ERROR: syntax error at or near "("
ALTER PROTOCOL demoprot update readfunc = read_from_file_immutable;
psql:/path/sql_file:1: ERROR: syntax error at or near "update"
ERROR: syntax error at or near "update"
LINE 1: ALTER PROTOCOL demoprot update readfunc = read_from_file_imm...
^
-- ERROR: syntax error at or near "update"
ALTER PROTOCOL demoprot trusted;
psql:/path/sql_file:1: ERROR: syntax error at or near "trusted"
ERROR: syntax error at or near "trusted"
LINE 1: ALTER PROTOCOL demoprot trusted;
^
-- ERROR: syntax error at or near "trusted"
RESET ROLE;
RESET
-- Test 96: Untrusted protocol - Superuser
-- Non-owner superuser does not have any limitation when using non-trusted protocol.
-- login as superuser huangh5
-- create untrusted protocol demoprot
DROP PROTOCOL IF EXISTS demoprot CASCADE;
DROP PROTOCOL
CREATE PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
CREATE PROTOCOL
-- Check the owner of trusted protocl demoprot is current superuser
select ptcname, ptctrusted
from pg_extprotocol join pg_user
......@@ -2057,7 +1818,6 @@ CREATE PROTOCOL
-- connect as a different superuser "demoprot_super"
SET ROLE demoprot_super;
SET
select user;
current_user
----------------
......@@ -2073,22 +1833,17 @@ SET
-- Verify superuser "demoprot_super" can create new ext table using untrusted protocol demoprot
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new;
NOTICE: table "exttabtest_r_new" does not exist, skipping
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE exttabtest_r_new(like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new;
NOTICE: table "exttabtest_w_new" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new(like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
-- Verify superuser demoprot_super can still export data via new created WET exttabtest_w_new
INSERT INTO exttabtest_w_new (SELECT * FROM exttabtest);
INSERT 0 100
-- Verify superuser demoprot_super can access new created RET exttabtest_r_new
select count(*) from exttabtest_r_new;
count
......@@ -2097,41 +1852,32 @@ INSERT 0 100
(1 row)
RESET ROLE;
RESET
-- Test 97: Non-trusted protocol - non-priv user
-- Non-privileged user cannot use non-trusted protocol to create external table.
-- With granted permissions on existing external table, Non-privileged user can access existing WET and RET
DROP EXTERNAL TABLE IF EXISTS exttabtest_r;
NOTICE: table "exttabtest_r" does not exist, skipping
DROP EXTERNAL TABLE
CREATE READABLE EXTERNAL TABLE exttabtest_r(like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w;
NOTICE: table "exttabtest_w" does not exist, skipping
DROP EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w(like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
-- As superuser, GRANT SELECT permission on RET
-- and INSERT permission on WET to on-privileged user "demoprot_nopriv"
-- to non-privileged user "demoprot_nopriv"
GRANT SELECT ON exttabtest_r TO demoprot_nopriv;
GRANT
GRANT INSERT ON exttabtest_w TO demoprot_nopriv;
GRANT
-- As superuser, GRANT SELECT permission on heap table "exttabtest"
-- to non-privileged user "demoprot_nopriv" so that this user
-- can try to create external table using format: (like exttabtest)
GRANT SELECT ON exttabtest TO demoprot_nopriv;
GRANT
-- connect as non-privileged user "demoprot_nopriv"
SET ROLE demoprot_nopriv;
SET
select user;
current_user
-----------------
......@@ -2167,7 +1913,6 @@ SET
(0 rows)
INSERT INTO exttabtest_w (SELECT * FROM exttabtest);
INSERT 0 100
-- Verify with SELECT permission granted, non-privileged user "demoprot_nopriv"
-- can load data via RET that was created using non-trusted protocol
select count(*) from exttabtest_r;
......@@ -2179,91 +1924,80 @@ INSERT 0 100
-- Verify non-privileged user "demoprot_nopriv" cannot create new ext table
-- using untrusted protocol demoprot
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new2;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_new2" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_r_new2" does not exist, skipping
CREATE READABLE EXTERNAL TABLE exttabtest_r_new2 (like exttabtest)
LOCATION('demoprot://exttabtest.txt')
FORMAT 'text';
psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot
ERROR: permission denied for external protocol demoprot
-- Test 98: Trusted Protocol - Negative Tests
-- create protocol using incorrect keyword TRUST instead of TRUSTED
DROP PROTOCOL IF EXISTS demoprot_trusted_bad;
psql:/path/sql_file:1: NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping
DROP PROTOCOL
NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping
CREATE TRUST PROTOCOL demoprot_trusted_bad (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
psql:/path/sql_file:1: ERROR: syntax error at or near "TRUST"
ERROR: syntax error at or near "TRUST"
LINE 1: CREATE TRUST PROTOCOL demoprot_trusted_bad (
^
-- create protocol using incorrect keyword UNTRUSTED
DROP PROTOCOL IF EXISTS demoprot_trusted_bad;
psql:/path/sql_file:1: NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping
DROP PROTOCOL
NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping
CREATE UNTRUSTED PROTOCOL demoprot_trusted_bad (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
psql:/path/sql_file:1: ERROR: syntax error at or near "UNTRUSTED"
ERROR: syntax error at or near "UNTRUSTED"
LINE 1: CREATE UNTRUSTED PROTOCOL demoprot_trusted_bad (
^
-- create protocol using TRUSTED at wrong position
DROP PROTOCOL IF EXISTS demoprot_trusted_bad;
psql:/path/sql_file:1: NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping
DROP PROTOCOL
NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping
CREATE PROTOCOL TRUSTED demoprot_trusted_bad (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
psql:/path/sql_file:1: ERROR: syntax error at or near "demoprot_trusted_bad"
ERROR: syntax error at or near "demoprot_trusted_bad"
LINE 1: CREATE PROTOCOL TRUSTED demoprot_trusted_bad (
^
-- create protocol using TRUSTED at wrong position
DROP PROTOCOL IF EXISTS demoprot_trusted_bad;
psql:/path/sql_file:1: NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping
DROP PROTOCOL
NOTICE: protocol "demoprot_trusted_bad" does not exist, skipping
CREATE PROTOCOL demoprot_trusted_bad TRUSTED (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
psql:/path/sql_file:1: ERROR: syntax error at or near "TRUSTED"
ERROR: syntax error at or near "TRUSTED"
LINE 1: CREATE PROTOCOL demoprot_trusted_bad TRUSTED (
^
RESET ROLE;
RESET
-- Test 99: Trusted protocol - Grant All
-- Grant all permissions ON trusted protocol to non-privileged user
-- Non-privileged user can use trusted protocol to create external table.
-- connect as superuser
-- create trusted protocol demoprot
DROP PROTOCOL IF EXISTS demoprot CASCADE;
NOTICE: drop cascades to external table exttabtest_w
NOTICE: drop cascades to external table exttabtest_r
NOTICE: drop cascades to external table exttabtest_w_new
NOTICE: drop cascades to external table exttabtest_r_new
DROP PROTOCOL
CREATE TRUSTED PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
CREATE PROTOCOL
-- As superuser, GRANT SELECT permission on heap table "exttabtest"
-- to non-privileged user "demoprot_nopriv" so that this user
-- can try to create external table using format: (like exttabtest)
GRANT SELECT ON exttabtest TO demoprot_nopriv;
GRANT
-- Drop existing WET and RET
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new;
NOTICE: table "exttabtest_r_new" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new;
NOTICE: table "exttabtest_w_new" does not exist, skipping
DROP EXTERNAL TABLE
-- As superuser, GRANT ALL privileges on protocol to non-privileged user demoprot_nopriv
GRANT ALL ON PROTOCOL demoprot TO demoprot_nopriv;
GRANT
-- login as non-privileged user "demoprot_nopriv"
SET ROLE demoprot_nopriv;
SET
select user;
current_user
-----------------
......@@ -2275,12 +2009,10 @@ SET
CREATE READABLE EXTERNAL TABLE exttabtest_r_new (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
-- Verify non-privileged user "demoprot_nopriv" can export data via new created WET exttabtest_w_new
SELECT * FROM clean_exttabtest_files;
stdout
......@@ -2288,7 +2020,6 @@ CREATE EXTERNAL TABLE
(0 rows)
INSERT INTO exttabtest_w_new (SELECT * FROM exttabtest);
INSERT 0 100
-- Verify non-privileged user "demoprot_nopriv" can load data via new created RET exttabtest_r_new
select count(*) from exttabtest_r_new;
count
......@@ -2297,7 +2028,6 @@ INSERT 0 100
(1 row)
RESET ROLE;
RESET
-- Test 99a: Trusted protocol - Revoke All
-- Revoke all permissions ON trusted protocol from non-privileged user
-- connect as superuser
......@@ -2305,44 +2035,39 @@ RESET
DROP PROTOCOL IF EXISTS demoprot CASCADE;
NOTICE: drop cascades to external table exttabtest_w_new
NOTICE: drop cascades to external table exttabtest_r_new
DROP PROTOCOL
CREATE TRUSTED PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
CREATE PROTOCOL
-- As superuser, GRANT SELECT permission on heap table "exttabtest"
-- to non-privileged user "demoprot_nopriv" so that this user
-- can try to create external table using format: (like exttabtest)
GRANT SELECT ON exttabtest TO demoprot_nopriv;
GRANT
-- As superuser, REVOKE ALL privileges on protocol from non-privileged user demoprot_nopriv
-- Both SELECT and INSERT permissions are revoked
REVOKE ALL ON PROTOCOL demoprot FROM demoprot_nopriv;
NOTICE: no privileges could be revoked from role demoprot_nopriv on object demoprot
REVOKE
-- login as non-privileged user "demoprot_nopriv"
SET ROLE demoprot_nopriv;
SET
select user;
current_user
-----------------
demoprot_nopriv
(1 row)
-- Verify after permissions have been revoked
-- non-privileged user "demoprot_nopriv" cannot create new ext table
-- using the trusted protocol demoprot
CREATE READABLE EXTERNAL TABLE exttabtest_r_new (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text';
psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot
ERROR: permission denied for external protocol demoprot
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot
ERROR: permission denied for external protocol demoprot
RESET ROLE;
RESET
-- Test 101: Trusted protocol - Grant Select
-- Grant SELECT permission ON trusted protocol to non-privileged user
-- Non-privileged user can use trusted protocol to create readable external table.
......@@ -2351,41 +2076,31 @@ RESET
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
SELECT * FROM clean_exttabtest_files;
stdout
--------
(0 rows)
INSERT INTO exttabtest_w (SELECT * FROM exttabtest);
INSERT 0 100
-- As superuser, demoport is created as a trusted readonly protocol
DROP PROTOCOL IF EXISTS demoprot CASCADE;
NOTICE: drop cascades to external table exttabtest_w
DROP PROTOCOL
CREATE TRUSTED PROTOCOL demoprot (
readfunc = read_from_file_stable
);
CREATE PROTOCOL
-- As superuser, GRANT SELECT permission on heap table "exttabtest"
-- to non-privileged user "demoprot_nopriv" so that this user
-- can try to create external table using format: (like exttabtest)
GRANT SELECT ON exttabtest TO demoprot_nopriv;
GRANT
-- Drop existing WET and RET
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new;
NOTICE: table "exttabtest_r_new" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new;
NOTICE: table "exttabtest_w_new" does not exist, skipping
DROP EXTERNAL TABLE
-- As superuser, GRANT SELECT permission on read protocol to non-privileged user demoprot_nopriv
GRANT SELECT ON PROTOCOL demoprot TO demoprot_nopriv;
GRANT
-- login as non-privileged user "demoprot_nopriv"
SET ROLE demoprot_nopriv;
SET
select user;
current_user
-----------------
......@@ -2397,13 +2112,12 @@ SET
CREATE READABLE EXTERNAL TABLE exttabtest_r_new (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text';
CREATE EXTERNAL TABLE
-- Verify non-privileged user "demoprot_nopriv" cannot create new writable ext table
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot
ERROR: permission denied for external protocol demoprot
-- Verify non-privileged user "demoprot_nopriv" can load data via new created RET exttabtest_r_new
select count(*) from exttabtest_r_new;
count
......@@ -2412,31 +2126,25 @@ psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot
(1 row)
RESET ROLE;
RESET
-- Test 102: Trusted protocol - Revoke Select
-- Revoke SELECT permission ON trusted protocol from non-privileged user
-- connect as superuser
-- create trusted protocol demoprot
DROP PROTOCOL IF EXISTS demoprot CASCADE;
NOTICE: drop cascades to external table exttabtest_r_new
DROP PROTOCOL
CREATE TRUSTED PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
CREATE PROTOCOL
-- As superuser, GRANT SELECT permission on heap table "exttabtest"
-- to non-privileged user "demoprot_nopriv" so that this user
-- can try to create external table using format: (like exttabtest)
GRANT SELECT ON exttabtest TO demoprot_nopriv;
GRANT
-- As superuser, REVOKE SElECT privilege on protocol from non-privileged user demoprot_nopriv
REVOKE SELECT ON PROTOCOL demoprot FROM demoprot_nopriv;
NOTICE: no privileges could be revoked from role demoprot_nopriv on object demoprot
REVOKE
-- login as non-privileged user "demoprot_nopriv"
SET ROLE demoprot_nopriv;
SET
select user;
current_user
-----------------
......@@ -2449,36 +2157,28 @@ SET
CREATE READABLE EXTERNAL TABLE exttabtest_r_new2 (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text';
psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot
ERROR: permission denied for external protocol demoprot
RESET ROLE;
RESET
-- Test 103: Trusted protocol - Grant Insert
-- Grant INSERT permission ON trusted protocol to non-privileged user
-- Non-privileged user can use trusted protocol to create writable external table.
-- As superuser, demoport is created as a trusted readonly protocol
DROP PROTOCOL IF EXISTS demoprot CASCADE;
DROP PROTOCOL
CREATE TRUSTED PROTOCOL demoprot (
writefunc = write_to_file_stable
);
CREATE PROTOCOL
-- As superuser, GRANT SELECT permission on heap table "exttabtest"
-- to non-privileged user "demoprot_nopriv" so that this user
-- can try to create external table using format: (like exttabtest)
GRANT SELECT ON exttabtest TO demoprot_nopriv;
GRANT
-- Drop existing WET and RET
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new;
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_new" does not exist, skipping
DROP EXTERNAL TABLE
NOTICE: table "exttabtest_w_new" does not exist, skipping
-- As superuser, GRANT INSERT permission on read protocol to non-privileged user demoprot_nopriv
GRANT INSERT ON PROTOCOL demoprot TO demoprot_nopriv;
GRANT
-- login as non-privileged user "demoprot_nopriv"
SET ROLE demoprot_nopriv;
SET
select user;
current_user
-----------------
......@@ -2490,13 +2190,12 @@ SET
CREATE READABLE EXTERNAL TABLE exttabtest_r_new (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text';
psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot
ERROR: permission denied for external protocol demoprot
-- Verify non-privileged user "demoprot_nopriv" can create new writable ext table
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_new (like exttabtest)
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
CREATE EXTERNAL TABLE
-- Verify non-privileged user "demoprot_nopriv" can export data via new created WET exttabtest_w_new
SELECT * FROM clean_exttabtest_files;
stdout
......@@ -2504,32 +2203,26 @@ CREATE EXTERNAL TABLE
(0 rows)
INSERT INTO exttabtest_w_new (SELECT * FROM exttabtest);
INSERT 0 100
RESET ROLE;
RESET
-- Test 104: Trusted protocol - Revoke Insert
-- Revoke INSERT permission ON trusted protocol from non-privileged user
-- connect as superuser
-- create trusted protocol demoprot
DROP PROTOCOL IF EXISTS demoprot CASCADE;
NOTICE: drop cascades to external table exttabtest_w_new
DROP PROTOCOL
CREATE TRUSTED PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
CREATE PROTOCOL
-- As superuser, GRANT SELECT permission on heap table "exttabtest"
-- to non-privileged user "demoprot_nopriv" so that this user
-- can try to create external table using format: (like exttabtest)
GRANT SELECT ON exttabtest TO demoprot_nopriv;
GRANT
-- As superuser, REVOKE INSERT privilege on protocol from non-privileged user demoprot_nopriv
REVOKE INSERT ON PROTOCOL demoprot FROM demoprot_nopriv;
REVOKE
NOTICE: no privileges could be revoked from role demoprot_nopriv on object demoprot
-- login as non-privileged user "demoprot_nopriv"
SET ROLE demoprot_nopriv;
SET
select user;
current_user
-----------------
......@@ -2543,6 +2236,5 @@ SET
LOCATION('demoprot://exttabtest_new.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
psql:/path/sql_file:1: ERROR: permission denied for external protocol demoprot
ERROR: permission denied for external protocol demoprot
RESET ROLE;
RESET
contrib/extprotocol/expected/setup.out 0 → 100644
浏览文件 @ f8734d9d
CREATE SCHEMA exttableext;
GRANT ALL ON SCHEMA exttableext TO PUBLIC;
SET search_path TO 'exttableext';
-- Create an example table exttabtest that will be used as source table
CREATE TABLE exttabtest(
id int,
name varchar(20),
value1 int,
value2 int
)
DISTRIBUTED BY (id);
-- Loading 100 records
-- Use only 100 rows for easy to verify the results
-- In order to test multiple data buffers and related edge cases, more data (at least several MBs or more)
-- will be used, as in Performance tests 1M and 100M test cases
\echo 'loading data...'
loading data...
INSERT INTO exttabtest SELECT i, 'name'||i, i*2, i*3 FROM generate_series(1,100) i;
-- Test 1: create read and write functions based on example gpextprotocol.so
-- Note: Only STABLE is supported for protocol, though this has not been enforced at the time of testing
CREATE OR REPLACE FUNCTION write_to_file_stable() RETURNS integer AS
'$libdir/gpextprotocol.so', 'demoprot_export' LANGUAGE C STABLE;
CREATE OR REPLACE FUNCTION read_from_file_stable() RETURNS integer AS
'$libdir/gpextprotocol.so', 'demoprot_import' LANGUAGE C STABLE;
-- Check pg_proc catalog table for new created functions
SELECT proname, prolang,proisstrict,provolatile,pronargs,prorettype,prosrc,proacl FROM pg_proc
WHERE proname like 'write_to_file%'
or proname like 'read_from_file%'
ORDER BY proname;
proname | prolang | proisstrict | provolatile | pronargs | prorettype | prosrc | proacl
-----------------------+---------+-------------+-------------+----------+------------+-----------------+--------
read_from_file_stable | 13 | f | s | 0 | 23 | demoprot_import |
write_to_file_stable | 13 | f | s | 0 | 23 | demoprot_export |
(2 rows)
-- Test 2: create bi-directional protocol (read and write) using STABLE functions
DROP PROTOCOL IF EXISTS demoprot;
NOTICE: protocol "demoprot" does not exist, skipping
CREATE PROTOCOL demoprot (
readfunc = read_from_file_stable,
writefunc = write_to_file_stable
);
-- Check dependency: pg_depend table
select count(*) from pg_depend
where objid in (
select oid from pg_extprotocol where ptcname='demoprot');
count
-------
2
(1 row)
-- Check pg_extprotocol for new created protocol
select extprot.ptcname, proc1.proname readfunc, proc2.proname writefunc
from pg_extprotocol extprot, pg_proc proc1, pg_proc proc2
where extprot.ptcname='demoprot'
and extprot.ptcreadfn=proc1.oid
and extprot.ptcwritefn=proc2.oid;
ptcname | readfunc | writefunc
----------+-----------------------+----------------------
demoprot | read_from_file_stable | write_to_file_stable
(1 row)
DROP EXTERNAL TABLE IF EXISTS clean_exttabtest_files;
NOTICE: table "clean_exttabtest_files" does not exist, skipping
CREATE EXTERNAL WEB TABLE clean_exttabtest_files(stdout text) EXECUTE 'rm -f exttabtest*' ON ALL FORMAT 'text';
GRANT ALL ON clean_exttabtest_files TO PUBLIC;
SELECT * FROM clean_exttabtest_files;
stdout
--------
(0 rows)
src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/exttableext.sql → contrib/extprotocol/sql/exttableext.sql
浏览文件 @ f8734d9d
......@@ -27,9 +27,11 @@ SET search_path TO 'exttableext';
EXCEPT ALL
SELECT * FROM exttabtest;
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from exttabtest_r
GROUP BY 1 ORDER BY 1;
-- verify data should be evenly distributed
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
-- Test 4.1: create uni-directional write protocol
-- create WET using created protocol
......@@ -101,9 +103,11 @@ SET search_path TO 'exttableext';
EXCEPT ALL
SELECT * FROM exttabtest;
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from exttabtest_r_dist
GROUP BY 1 ORDER BY 1;
-- verify data should be evenly distributed
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_dist group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
-- Test 6: using two urls and using CSV format
......@@ -128,9 +132,11 @@ SET search_path TO 'exttableext';
EXCEPT ALL
SELECT * FROM exttabtest;
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from exttabtest_r_2url
GROUP BY 1 ORDER BY 1;
-- verify data should be evenly distributed
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_2url group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
-- Check the output file at each segments
-- ! gpssh -f allsegs ls -l /data/hhuang/MAIN/main_debug/primary/gpseg*/exttabtest_2url*.csv
......@@ -157,9 +163,11 @@ SET search_path TO 'exttableext';
EXCEPT ALL
SELECT * FROM exttabtest;
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from exttabtest_r_2url
GROUP BY 1 ORDER BY 1;
-- verify data should be evenly distributed
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_2url group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
-- Checking the output files on segments
-- ! gpssh -f allsegs ls -l /data/hhuang/MAIN/main_debug/primary/gpseg*/exttabtest_2url*.txt
......@@ -295,32 +303,34 @@ SET search_path TO 'exttableext';
LOCATION('badprotocol://exttabtest.txt')
FORMAT 'text';
-- Test 20: Small dataset - 4 records
-- Test 20: Small dataset - 20 records
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_4records;
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_4records (like exttabtest)
LOCATION('demoprot://exttabtest_4records.txt')
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_20records;
CREATE WRITABLE EXTERNAL TABLE exttabtest_w_20records (like exttabtest)
LOCATION('demoprot://exttabtest_20records.txt')
FORMAT 'text'
DISTRIBUTED BY (id);
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_4records;
CREATE READABLE EXTERNAL TABLE exttabtest_r_4records (like exttabtest)
LOCATION('demoprot://exttabtest_4records.txt')
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_20records;
CREATE READABLE EXTERNAL TABLE exttabtest_r_20records (like exttabtest)
LOCATION('demoprot://exttabtest_20records.txt')
FORMAT 'text';
-- write to WET
INSERT INTO exttabtest_w_4records (SELECT * FROM exttabtest where id<=4);
INSERT INTO exttabtest_w_20records (SELECT * FROM exttabtest where id<=20);
-- read from RET
SELECT * FROM exttabtest_r_4records order by id;
SELECT * FROM exttabtest_r_20records order by id;
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from exttabtest_r_4records
GROUP BY 1 ORDER BY 1;
-- verify data should be evenly distributed
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from exttabtest_r_20records group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
-- Drop External Tables
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_4records;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_4records;
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_20records;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_20records;
-- Test 21: Small dataset - 1 record
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1record;
......@@ -597,7 +607,10 @@ DISTRIBUTED BY (id);
INSERT INTO formatsource SELECT 'name'||i, i, i*2, i*3 FROM generate_series(1,100) i;
-- Check data distribution
SELECT gp_segment_id, count(*) FROM formatsource GROUP BY 1 ORDER BY 1;
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from formatsource group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
-- Test 61: create STABLE read and write functions based on example gpformatter.so
-- Note: Only STABLE is supported for formatter.
......@@ -706,9 +719,12 @@ DROP FUNCTION formatter_import_todrop();
-- Read from RET
SELECT count(*) FROM format_r;
-- verify data should be evenly distributed
SELECT gp_segment_id, count(*) from format_r
GROUP BY 1 ORDER BY 1;
-- verify data should be evenly distributed
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from format_r group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
-- Test 64: Drop format function with external table using the function
DROP FUNCTION formatter_export_i(record);
......@@ -855,16 +871,21 @@ DROP FUNCTION formatter_import_todrop();
insert into format_long select 2.0,i from repeat('oxo2', 1000) i;
insert into format_long select 3.0,i from repeat('oxo3', 1000) i;
insert into format_long select 4.0,i from repeat('oxo4', 1000) i;
insert into format_long select 5.0,i from repeat('oxo5', 1000) i;
insert into format_long select 6.0,i from repeat('oxo6', 1000) i;
insert into format_long select 7.0,i from repeat('oxo7', 1000) i;
insert into format_long select 8.0,i from repeat('oxo8', 1000) i;
insert into format_long select 9.0,i from repeat('oxo9', 1000) i;
-- Check distribution is even
select gp_segment_id,count(*) from format_long
group by gp_segment_id
order by gp_segment_id;
with t as (
SELECT gp_segment_id as segid, count(*) as cnt from format_long group by gp_segment_id
)
select max(cnt) - min(cnt) > 20 from t;
-- Write to WET
-- insert should be successful
INSERT INTO format_long_w (SELECT * FROM format_long);
-- INSERT 0 4
-- Read from RET
select count(*) from format_long_r;
......@@ -915,15 +936,19 @@ DROP FUNCTION formatter_import_todrop();
insert into format_long select 2.0,i from repeat('oxox2', 1000) i;
insert into format_long select 3.0,i from repeat('oxox3', 1000) i;
insert into format_long select 4.0,i from repeat('oxox4', 1000) i;
insert into format_long select 5.0,i from repeat('oxox5', 1000) i;
insert into format_long select 6.0,i from repeat('oxox6', 1000) i;
insert into format_long select 7.0,i from repeat('oxox7', 1000) i;
insert into format_long select 8.0,i from repeat('oxox8', 1000) i;
insert into format_long select 9.0,i from repeat('oxox9', 1000) i;
-- Write to WET
-- insert should be successful
INSERT INTO format_long_w (SELECT * FROM format_long);
-- INSERT 0 4
-- Read from RET
select count(*) from format_long_r;
-- returns count = 4
-- returns count = 9
-- Test 73: Interlacing short and long record, testing FMT_NEED_MORE_DATA
-- When loading data from RET, long record may trigger FORMATTER_RETURN_NOTIFICATION(fcinfo, FMT_NEED_MORE_DATA).
-- Verify the data can be loaded successfully and should exactly match the source records.
......@@ -960,15 +985,19 @@ DROP FUNCTION formatter_import_todrop();
insert into format_long select 2.0,i from repeat('oxo2', 1000) i;
insert into format_long select 3.0,'oxo3';
insert into format_long select 4.0,i from repeat('oxo4', 1000) i;
insert into format_long select 5.0,'oxo5';
insert into format_long select 6.0,i from repeat('oxo6', 1000) i;
insert into format_long select 7.0,'oxo7';
insert into format_long select 8.0,i from repeat('oxo8', 1000) i;
insert into format_long select 9.0,'oxo9';
-- Write to WET
-- insert should be successful
INSERT INTO format_long_w (SELECT * FROM format_long);
-- INSERT 0 4
-- Read from RET
select count(*) from format_long_r;
-- returns count = 4
-- returns count = 9
-- read from RET, both should return 0
SELECT * FROM format_long_r
......@@ -1091,11 +1120,15 @@ DROP FUNCTION formatter_import_todrop();
insert into formatsource select 2,'oxo2';
insert into formatsource select 3,'oxo3';
insert into formatsource select 4,'oxo4';
insert into formatsource select 5,'oxo5';
insert into formatsource select 6,'oxo6';
insert into formatsource select 7,'oxo7';
insert into formatsource select 8,'oxo8';
insert into formatsource select 9,'oxo9';
-- Write to WET
-- insert should be successful
INSERT INTO format_w (SELECT * FROM formatsource);
-- INSERT 0 4
-- Read from RET
select id, value1 from format_r order by id;
......@@ -1130,11 +1163,15 @@ DROP FUNCTION formatter_import_todrop();
insert into formatsource values (2,null,null);
insert into formatsource values (3,null,null);
insert into formatsource values (4,null,null);
insert into formatsource values (5,null,null);
insert into formatsource values (6,null,null);
insert into formatsource values (7,null,null);
insert into formatsource values (8,null,null);
insert into formatsource values (9,null,null);
-- Write to WET
-- insert should be successful
INSERT INTO format_w (SELECT * FROM formatsource);
-- INSERT 0 4
-- Read from RET
select * from format_r where name is null order by id;
......
src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/setup/query00.sql → contrib/extprotocol/sql/setup.sql
浏览文件 @ f8734d9d
DROP SCHEMA IF EXISTS exttableext CASCADE;
CREATE SCHEMA exttableext;
GRANT ALL ON SCHEMA exttableext TO PUBLIC;
SET search_path TO 'exttableext';
-- Create an example table exttabtest that will be used as source table
-- Clean up existing external tables, functions
DROP TABLE IF EXISTS exttabtest;
DROP EXTERNAL TABLE IF EXISTS exttabtest_r;
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1m;
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1m_null;
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_2url;
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_circle;
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_dist;
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_invalid;
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new;
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_null;
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_uni;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1m;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1m_null;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_2url;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_5url;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_circle;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_dist;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_invalid;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_misspath;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_null;
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_uni;
DROP EXTERNAL TABLE IF EXISTS format_long_r;
DROP EXTERNAL TABLE IF EXISTS format_long_w;
DROP EXTERNAL TABLE IF EXISTS format_r;
DROP EXTERNAL TABLE IF EXISTS format_r_s1;
DROP EXTERNAL TABLE IF EXISTS format_r_s2;
DROP EXTERNAL TABLE IF EXISTS format_w;
DROP EXTERNAL TABLE IF EXISTS format_w_s1;
DROP EXTERNAL TABLE IF EXISTS format_w_s2;
DROP FUNCTION IF EXISTS formatter_export_s(record) CASCADE;
DROP FUNCTION IF EXISTS formatter_export_v(record) CASCADE;
DROP FUNCTION IF EXISTS formatter_import_s() CASCADE;
DROP FUNCTION IF EXISTS formatter_import_v() CASCADE;
DROP FUNCTION IF EXISTS read_from_file() CASCADE;
DROP FUNCTION IF EXISTS read_from_file_stable() CASCADE;
DROP FUNCTION IF EXISTS write_to_file_stable() CASCADE;
DROP FUNCTION IF EXISTS url_validator() CASCADE;
DROP FUNCTION IF EXISTS write_to_file() CASCADE;
DROP FUNCTION IF EXISTS write_to_file_stable() CASCADE;
CREATE TABLE exttabtest(
id int,
name varchar(20),
......
src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/expected/setup/query00.ans 已删除 100644 → 0
浏览文件 @ 6133a493
DROP SCHEMA IF EXISTS exttableext;
DROP SCHEMA
CREATE SCHEMA exttableext;
CREATE SCHEMA
GRANT ALL ON SCHEMA exttableext TO PUBLIC;
GRANT
SET search_path TO 'exttablext';
SET
-- Create an example table exttabtest that will be used as source table
-- Clean up existing external tables, functions
DROP TABLE IF EXISTS exttabtest;
psql:/path/sql_file:1: NOTICE: table "exttabtest" does not exist, skipping
DROP TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1m;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_1m" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_1m_null;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_1m_null" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_2url;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_2url" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_circle;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_circle" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_dist;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_dist" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_invalid;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_invalid" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_new;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_new" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_null;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_null" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_r_uni;
psql:/path/sql_file:1: NOTICE: table "exttabtest_r_uni" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1m;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_1m" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_1m_null;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_1m_null" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_2url;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_2url" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_5url;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_5url" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_circle;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_circle" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_dist;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_dist" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_invalid;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_invalid" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_misspath;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_misspath" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_new;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_new" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_null;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_null" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS exttabtest_w_uni;
psql:/path/sql_file:1: NOTICE: table "exttabtest_w_uni" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS format_long_r;
psql:/path/sql_file:1: NOTICE: table "format_long_r" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS format_long_w;
psql:/path/sql_file:1: NOTICE: table "format_long_w" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS format_r;
psql:/path/sql_file:1: NOTICE: table "format_r" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS format_r_s1;
psql:/path/sql_file:1: NOTICE: table "format_r_s1" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS format_r_s2;
psql:/path/sql_file:1: NOTICE: table "format_r_s2" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS format_w;
psql:/path/sql_file:1: NOTICE: table "format_w" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS format_w_s1;
psql:/path/sql_file:1: NOTICE: table "format_w_s1" does not exist, skipping
DROP EXTERNAL TABLE
DROP EXTERNAL TABLE IF EXISTS format_w_s2;
psql:/path/sql_file:1: NOTICE: table "format_w_s2" does not exist, skipping
DROP EXTERNAL TABLE
DROP FUNCTION IF EXISTS formatter_export_s(record) CASCADE;
psql:/path/sql_file:1: NOTICE: function formatter_export_s(record) does not exist, skipping
DROP FUNCTION
DROP FUNCTION IF EXISTS formatter_export_v(record) CASCADE;
psql:/path/sql_file:1: NOTICE: function formatter_export_v(record) does not exist, skipping
DROP FUNCTION
DROP FUNCTION IF EXISTS formatter_import_s() CASCADE;
psql:/path/sql_file:1: NOTICE: function formatter_import_s() does not exist, skipping
DROP FUNCTION
DROP FUNCTION IF EXISTS formatter_import_v() CASCADE;
psql:/path/sql_file:1: NOTICE: function formatter_import_v() does not exist, skipping
DROP FUNCTION
DROP FUNCTION IF EXISTS read_from_file() CASCADE;
psql:/path/sql_file:1: NOTICE: function read_from_file() does not exist, skipping
DROP FUNCTION
DROP FUNCTION IF EXISTS read_from_file_stable() CASCADE;
psql:/path/sql_file:1: NOTICE: function read_from_file_stable() does not exist, skipping
DROP FUNCTION
DROP FUNCTION IF EXISTS write_to_file_stable() CASCADE;
psql:/path/sql_file:1: NOTICE: function write_to_file_stable() does not exist, skipping
DROP FUNCTION
DROP FUNCTION IF EXISTS url_validator() CASCADE;
psql:/path/sql_file:1: NOTICE: function url_validator() does not exist, skipping
DROP FUNCTION
DROP FUNCTION IF EXISTS write_to_file() CASCADE;
psql:/path/sql_file:1: NOTICE: function write_to_file() does not exist, skipping
DROP FUNCTION
DROP FUNCTION IF EXISTS write_to_file_stable() CASCADE;
psql:/path/sql_file:1: NOTICE: function write_to_file_stable() does not exist, skipping
DROP FUNCTION
CREATE TABLE exttabtest(
id int,
name varchar(20),
value1 int,
value2 int
)
DISTRIBUTED BY (id);
CREATE TABLE
-- Loading 100 records
-- Use only 100 rows for easy to verify the results
-- In order to test multiple data buffers and related edge cases, more data (at least several MBs or more)
-- will be used, as in Performance tests 1M and 100M test cases
\echo 'loading data...'
loading data...
INSERT INTO exttabtest SELECT i, 'name'||i, i*2, i*3 FROM generate_series(1,100) i;
INSERT 0 100
DROP EXTERNAL TABLE IF EXISTS clean_exttabtest_files;
DROP TABLE
CREATE EXTERNAL WEB TABLE clean_exttabtest_files(stdout text) EXECUTE 'rm -f exttabtest*' ON ALL FORMAT 'text';
CREATE TABLE
GRANT ALL ON clean_exttabtest_files TO PUBLIC;
GRANT
SELECT * FROM clean_exttabtest_files;
stdout
--------
(0 rows)
src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/init_file 已删除 100644 → 0
浏览文件 @ 6133a493
-- start_matchsubs
# Change things like "psql: /Users/mglkey/cdbfast/main/foo/bar.sql:123:" to "PATH"
m/psql:\/.*:\d+:/
s/psql:\/.*:\d+:/psql:PATH:/
-- end_matchsubs
src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/sql/setup/removeOutputFile.sql 已删除 100644 → 0
浏览文件 @ 6133a493
-- WET using example protocol demoprot will export data files to data directory on primary segments
-- If NOT cleaned, these files will break data replication (between primary and mirror).
-- Therefore it is necessary to clean up generated data files after each test case run.
--
-- query to construct command to remove the generated data file from data directory
SELECT 'gpssh -h '|| gpsc.hostname ||' rm -f '|| fse.fselocation || '/exttabtest*'
FROM pg_filespace_entry fse, gp_segment_configuration gpsc
WHERE gpsc.dbid = fse.fsedbid and gpsc.role = 'p' and gpsc.content > -1
ORDER BY gpsc.hostname, fse.fselocation;
src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/exttab/exttableext/test_exttable.py 已删除 100755 → 0
浏览文件 @ 6133a493
"""
Copyright (C) 2004-2015 Pivotal Software, Inc. All rights reserved.
This program and the accompanying materials are made available under
the terms of the under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
from mpp.models import SQLTestCase
from tinctest.lib import local_path, run_shell_command
class ExternalTableExtensionTests(SQLTestCase):
"""
@tags list_mgmt_expand
@product_version gpdb: [4.3-]
"""
sql_dir = 'sql/'
ans_dir = 'expected/'
def tearDown(self):
# execute commands to remove output files created in primary segment's data directories
# This cleans up all the files exported by demoprot on the primary segments.
# expects removeOutputFile.sql in sql/setup to be run and its corredponfing out file available in
# out_dir/setup/removeOutputFile.out
out_file = os.path.join(self.get_out_dir(), 'setup', 'removeOutputFile.out')
with open(out_file) as f:
for line in f:
if (line.find('gpssh ')>=0 and line.find('SELECT')<0):
ok = run_shell_command(line)
if not ok:
raise Exception('Output file remove operation error: %s' %line)
super(ExternalTableExtensionTests, self).tearDown()
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或