From 712705e1e8f58897274a8a23dcc163cd7fe99364 Mon Sep 17 00:00:00 2001 From: Adam Lee Date: Fri, 15 Jul 2016 16:55:24 +0800 Subject: [PATCH] s3ext: update regress cases (#950) 1. speed up running by specifying proper files with sufficient size 2. add a pair of tests which connect to the same S3 file in parallel 3. revise sql scripts to make them neat Signed-off-by: Adam Lee Signed-off-by: Kuien Liu --- .../gps3ext/regress/gpcheckcloud_regress.sh | 30 +-- .../gps3ext/regress/input/1_01_normal.source | 15 +- .../regress/input/1_02_log_error.source | 15 +- .../regress/input/1_03_bad_data.source | 15 +- .../regress/input/1_04_empty_prefix.source | 15 +- .../regress/input/1_05_one_line.source | 13 +- .../regress/input/1_06_1correct_1wrong.source | 16 +- .../regress/input/1_07_huge_bad_data.source | 11 +- .../input/1_08_huge_correct_data.source | 11 +- .../regress/input/1_09_partition.source | 10 +- .../regress/input/1_10_all_regions.source | 113 ++++++---- .../regress/input/1_11_gzipped_data.source | 22 +- .../regress/input/1_12_no_prefix.source | 32 +-- .../regress/input/1_13_parallel1.source | 21 ++ .../regress/input/1_13_parallel2.source | 21 ++ .../input/1_14_thousands_of_files.source | 17 ++ .../regress/input/2_01_invalid_syntax.source | 13 +- .../regress/input/2_02_invalid_region.source | 13 +- .../regress/input/2_03_invalid_config.source | 13 +- .../regress/input/2_04_invalid_header.source | 13 +- .../gps3ext/regress/output/1_01_normal.source | 32 +-- .../regress/output/1_02_log_error.source | 16 +- .../regress/output/1_03_bad_data.source | 26 +-- .../regress/output/1_04_empty_prefix.source | 16 +- .../regress/output/1_05_one_line.source | 14 +- .../output/1_06_1correct_1wrong.source | 22 +- .../regress/output/1_07_huge_bad_data.source | 18 +- .../output/1_08_huge_correct_data.source | 18 +- .../regress/output/1_09_partition.source | 24 +- .../regress/output/1_10_all_regions.source | 211 ++++++++++-------- .../regress/output/1_11_gzipped_data.source | 40 ++-- .../regress/output/1_12_no_prefix.source | 86 +------ .../regress/output/1_13_parallel1.source | 24 ++ .../regress/output/1_13_parallel2.source | 23 ++ .../output/1_14_thousands_of_files.source | 17 ++ .../regress/output/2_01_invalid_syntax.source | 21 +- .../regress/output/2_02_invalid_region.source | 18 +- .../regress/output/2_03_invalid_config.source | 18 +- .../regress/output/2_04_invalid_header.source | 18 +- .../gps3ext/regress/s3_regress_schedule | 2 + 40 files changed, 548 insertions(+), 545 deletions(-) create mode 100644 gpAux/extensions/gps3ext/regress/input/1_13_parallel1.source create mode 100644 gpAux/extensions/gps3ext/regress/input/1_13_parallel2.source create mode 100644 gpAux/extensions/gps3ext/regress/input/1_14_thousands_of_files.source create mode 100644 gpAux/extensions/gps3ext/regress/output/1_13_parallel1.source create mode 100644 gpAux/extensions/gps3ext/regress/output/1_13_parallel2.source create mode 100644 gpAux/extensions/gps3ext/regress/output/1_14_thousands_of_files.source diff --git a/gpAux/extensions/gps3ext/regress/gpcheckcloud_regress.sh b/gpAux/extensions/gps3ext/regress/gpcheckcloud_regress.sh index bcf4916dd1..4cc71caf82 100755 --- a/gpAux/extensions/gps3ext/regress/gpcheckcloud_regress.sh +++ b/gpAux/extensions/gps3ext/regress/gpcheckcloud_regress.sh @@ -3,16 +3,16 @@ EXIT_CODE=0 GPCHECKCLOUD=./bin/gpcheckcloud/gpcheckcloud -PREFIX[0]="http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0014" -PREFIX[1]="http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0016" -PREFIX[2]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0014" -PREFIX[3]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0016" -PREFIX[4]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/gzipped/data0001.gz" -PREFIX[5]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/threebytes/" -PREFIX[6]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/" -PREFIX[7]="s3://s3-us-east-1.amazonaws.com/useast1.s3test.pivotal.io/small17/" -PREFIX[8]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/gzipped/" -PREFIX[9]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/gzipped_normal1/" +PREFIX[0]="http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/small17/data0014" +PREFIX[1]="http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/small17/data0016" +PREFIX[2]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/small17/data0014" +PREFIX[3]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/small17/data0016" +PREFIX[4]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped/data0001.gz" +PREFIX[5]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/threebytes/" +PREFIX[6]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/small17/" +PREFIX[7]="s3://s3-us-east-1.amazonaws.com/us-east-1.s3test.pivotal.io/regress/small17/" +PREFIX[8]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped/" +PREFIX[9]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_normal1/" HASHSUM[0]="68c4a63b721e7af0ae945ce109ca87ad" HASHSUM[1]="0fd502a303eb8f138f5916ec357721b1" @@ -26,13 +26,13 @@ HASHSUM[8]="7b2260e9a3a3f26e84aa28dc2124f68f" HASHSUM[9]="eacb7b210d3f7703ee06d16f520b103e" #HUGE FILES -PREFIX[10]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/airlinedata1.csv" -PREFIX[11]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/" -PREFIX[12]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/gzipped_normal2/" +PREFIX[10]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/hugefile/airlinedata1.csv" +PREFIX[11]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_normal2/" +PREFIX[12]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/5120files/" HASHSUM[10]="f5811ad92c994f1d6913d5338575fe38" -HASHSUM[11]="75baaa39f2b1544ed8af437c2cad86b7" -HASHSUM[12]="a930794bc885bccf6eed45bd40367a7d" +HASHSUM[11]="a930794bc885bccf6eed45bd40367a7d" +HASHSUM[12]="71277a7004842241191cc65146be24f6" for ((i=0; i<${#PREFIX[@]}; i++)) do diff --git a/gpAux/extensions/gps3ext/regress/input/1_01_normal.source b/gpAux/extensions/gps3ext/regress/input/1_01_normal.source index 09b362b7ad..15c1fada13 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_01_normal.source +++ b/gpAux/extensions/gps3ext/regress/input/1_01_normal.source @@ -1,20 +1,17 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; -SELECT sum(open) FROM s3example; -SELECT avg(open) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -DROP EXTERNAL TABLE s3example; +SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress; + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/1_02_log_error.source b/gpAux/extensions/gps3ext/regress/input/1_02_log_error.source index a6eed04c29..45617c9c85 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_02_log_error.source +++ b/gpAux/extensions/gps3ext/regress/input/1_02_log_error.source @@ -1,19 +1,18 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'text' ( DELIMITER ',' NULL '') LOG ERRORS SEGMENT REJECT LIMIT 5; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; -SELECT gp_read_error_log('s3example'); +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'text' ( DELIMITER ',' NULL '') LOG ERRORS SEGMENT REJECT LIMIT 5; -DROP EXTERNAL TABLE s3example; +SELECT count(*) FROM s3regress; +SELECT gp_read_error_log('s3regress'); + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/1_03_bad_data.source b/gpAux/extensions/gps3ext/regress/input/1_03_bad_data.source index e2cf817e98..d45ff30d02 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_03_bad_data.source +++ b/gpAux/extensions/gps3ext/regress/input/1_03_bad_data.source @@ -1,22 +1,21 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/baddata/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; + +SELECT count(*) FROM s3regress; -- start_ignore -SELECT gp_read_error_log('s3example'); +SELECT gp_read_error_log('s3regress'); -- end_ignore -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/1_04_empty_prefix.source b/gpAux/extensions/gps3ext/regress/input/1_04_empty_prefix.source index 4142130a83..f7ebfd51b1 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_04_empty_prefix.source +++ b/gpAux/extensions/gps3ext/regress/input/1_04_empty_prefix.source @@ -1,19 +1,18 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/emptyfile/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; -SELECT gp_read_error_log('s3example'); +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/emptyfile/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; -DROP EXTERNAL TABLE s3example; +SELECT count(*) FROM s3regress; +SELECT gp_read_error_log('s3regress'); + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/1_05_one_line.source b/gpAux/extensions/gps3ext/regress/input/1_05_one_line.source index 994f1b4ed6..c2dcf05ff5 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_05_one_line.source +++ b/gpAux/extensions/gps3ext/regress/input/1_05_one_line.source @@ -1,18 +1,17 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/oneline/ config=/home/gpadmin/s3.conf') format 'csv'; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/oneline/ config=/home/gpadmin/s3.conf') format 'csv'; -DROP EXTERNAL TABLE s3example; +SELECT count(*) FROM s3regress; + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/1_06_1correct_1wrong.source b/gpAux/extensions/gps3ext/regress/input/1_06_1correct_1wrong.source index b00599c10b..959503c539 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_06_1correct_1wrong.source +++ b/gpAux/extensions/gps3ext/regress/input/1_06_1correct_1wrong.source @@ -1,21 +1,21 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/1correct1wrong/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; +DROP EXTERNAL TABLE s3regress; + +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/1correct1wrong/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; + +SELECT count(*) FROM s3regress; -SELECT count(*) FROM s3example; -- start_ignore -SELECT gp_read_error_log('s3example'); +SELECT gp_read_error_log('s3regress'); -- end_ignore -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/1_07_huge_bad_data.source b/gpAux/extensions/gps3ext/regress/input/1_07_huge_bad_data.source index 46891758a0..fec9c42c4c 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_07_huge_bad_data.source +++ b/gpAux/extensions/gps3ext/regress/input/1_07_huge_bad_data.source @@ -1,17 +1,16 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/hugefile/airlinedata2.csv config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; -DROP EXTERNAL TABLE s3example; +SELECT count(*) FROM s3regress; + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/1_08_huge_correct_data.source b/gpAux/extensions/gps3ext/regress/input/1_08_huge_correct_data.source index efcbe56310..d167b49eca 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_08_huge_correct_data.source +++ b/gpAux/extensions/gps3ext/regress/input/1_08_huge_correct_data.source @@ -1,17 +1,16 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile_correct_data/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/hugefile_correct_data/airlinedata1.csv config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; -DROP EXTERNAL TABLE s3example; +SELECT count(*) FROM s3regress; + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/1_09_partition.source b/gpAux/extensions/gps3ext/regress/input/1_09_partition.source index ffb3987537..9d07e45475 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_09_partition.source +++ b/gpAux/extensions/gps3ext/regress/input/1_09_partition.source @@ -1,13 +1,11 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS -'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; + '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; +DROP EXTERNAL TABLE s3regress; CREATE TABLE stock (date text, time text, open float, high float, low float, volume int) DISTRIBUTED BY (date) PARTITION BY RANGE (volume) ( @@ -18,8 +16,8 @@ CREATE TABLE stock (date text, time text, open float, high float, low float, vol INSERT INTO stock (date, time, open, high, low, volume) VALUES (03/08/2011, 094022, 35.25, 35.24, 35.29, 20001); -create READABLE external table ext_stock000 (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +CREATE READABLE EXTERNAL TABLE ext_stock000 (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/xao config=/home/gpadmin/s3.conf') FORMAT 'csv'; ALTER TABLE stock ADD PARTITION stock000 START (0) INCLUSIVE END (10000) EXCLUSIVE; diff --git a/gpAux/extensions/gps3ext/regress/input/1_10_all_regions.source b/gpAux/extensions/gps3ext/regress/input/1_10_all_regions.source index 8264b05542..149627d0b0 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_10_all_regions.source +++ b/gpAux/extensions/gps3ext/regress/input/1_10_all_regions.source @@ -1,93 +1,110 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +DROP EXTERNAL TABLE s3regress; -\d s3example +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-east-1.amazonaws.com/us-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; -SELECT sum(open) FROM s3example; +\d s3regress +SELECT round(sum(open)) FROM s3regress; -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-ap-northeast-2.amazonaws.com/apnortheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example +DROP EXTERNAL TABLE s3regress; -SELECT sum(open) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-1.amazonaws.com/us-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-ap-southeast-1.amazonaws.com/apsoutheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +SELECT round(sum(open)) FROM s3regress; -\d s3example -SELECT sum(open) FROM s3example; +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-south-1.amazonaws.com/ap-south-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-ap-southeast-2.amazonaws.com/apsoutheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress -\d s3example +SELECT round(sum(open)) FROM s3regress; -SELECT sum(open) FROM s3example; +DROP EXTERNAL TABLE s3regress; -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-northeast-2.amazonaws.com/ap-northeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example +\d s3regress -SELECT sum(open) FROM s3example; +SELECT round(sum(open)) FROM s3regress; -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-eu-west-1.amazonaws.com/euwest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +DROP EXTERNAL TABLE s3regress; -\d s3example +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-southeast-1.amazonaws.com/ap-southeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; -SELECT sum(open) FROM s3example; +\d s3regress +SELECT round(sum(open)) FROM s3regress; -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example +DROP EXTERNAL TABLE s3regress; -SELECT sum(open) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-southeast-2.amazonaws.com/ap-southeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-east-1.amazonaws.com/useast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +SELECT round(sum(open)) FROM s3regress; -\d s3example -SELECT sum(open) FROM s3example; +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-northeast-1.amazonaws.com/ap-northeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-1.amazonaws.com/uswest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress -\d s3example +SELECT round(sum(open)) FROM s3regress; -SELECT sum(open) FROM s3example; -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; + +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-eu-central-1.amazonaws.com/eu-central-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; + +\d s3regress + +SELECT round(sum(open)) FROM s3regress; + + +DROP EXTERNAL TABLE s3regress; + +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-eu-west-1.amazonaws.com/eu-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; + +\d s3regress + +SELECT round(sum(open)) FROM s3regress; + + +DROP EXTERNAL TABLE s3regress; + +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-sa-east-1.amazonaws.com/sa-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; + +\d s3regress + +SELECT round(sum(open)) FROM s3regress; + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/1_11_gzipped_data.source b/gpAux/extensions/gps3ext/regress/input/1_11_gzipped_data.source index 8cecb4a1f8..5df1fdf5b0 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_11_gzipped_data.source +++ b/gpAux/extensions/gps3ext/regress/input/1_11_gzipped_data.source @@ -1,25 +1,23 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/gzipped_normal1/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; -SELECT sum(open) FROM s3example; -SELECT avg(open) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_normal1/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -drop external table s3example; -create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/gzipped_normal2/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; +SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress; -SELECT count(*) FROM s3example; +DROP EXTERNAL TABLE s3regress; -DROP EXTERNAL TABLE s3example; +CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_2002and2003/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; + +SELECT count(*) FROM s3regress; + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/1_12_no_prefix.source b/gpAux/extensions/gps3ext/regress/input/1_12_no_prefix.source index f6dc62d88c..7f6d92ac43 100644 --- a/gpAux/extensions/gps3ext/regress/input/1_12_no_prefix.source +++ b/gpAux/extensions/gps3ext/regress/input/1_12_no_prefix.source @@ -1,39 +1,17 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); +DROP EXTERNAL TABLE s3regress; -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io config=/home/gpadmin/s3.conf') FORMAT 'csv'; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-northeast-1.amazonaws.com/ap-northeast-1.s3test.pivotal.io config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example +SELECT round(sum(open)) FROM s3regress; -SELECT sum(open) FROM s3example; - - -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; - -\d s3example - -SELECT sum(open) FROM s3example; - - -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; - -\d s3example - -SELECT sum(open) FROM s3example; - -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/1_13_parallel1.source b/gpAux/extensions/gps3ext/regress/input/1_13_parallel1.source new file mode 100644 index 0000000000..d5bc3b5a63 --- /dev/null +++ b/gpAux/extensions/gps3ext/regress/input/1_13_parallel1.source @@ -0,0 +1,21 @@ +-- start_ignore +CREATE FUNCTION read_from_s3() RETURNS integer AS + '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; + +CREATE PROTOCOL s3 ( + readfunc = read_from_s3 +); +-- end_ignore + +DROP EXTERNAL TABLE parallel1; + +CREATE READABLE EXTERNAL TABLE parallel1 (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; + +SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM parallel1; + +DROP EXTERNAL TABLE parallel1; + +-- start_ignore +DROP PROTOCOL s3; +-- end_ignore diff --git a/gpAux/extensions/gps3ext/regress/input/1_13_parallel2.source b/gpAux/extensions/gps3ext/regress/input/1_13_parallel2.source new file mode 100644 index 0000000000..86e1bd725c --- /dev/null +++ b/gpAux/extensions/gps3ext/regress/input/1_13_parallel2.source @@ -0,0 +1,21 @@ +-- start_ignore +CREATE FUNCTION read_from_s3() RETURNS integer AS + '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; + +CREATE PROTOCOL s3 ( + readfunc = read_from_s3 +); +-- end_ignore + +DROP EXTERNAL TABLE parallel2; + +CREATE READABLE EXTERNAL TABLE parallel2 (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; + +SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM parallel2; + +DROP EXTERNAL TABLE parallel2; + +-- start_ignore +DROP PROTOCOL s3; +-- end_ignore diff --git a/gpAux/extensions/gps3ext/regress/input/1_14_thousands_of_files.source b/gpAux/extensions/gps3ext/regress/input/1_14_thousands_of_files.source new file mode 100644 index 0000000000..238bc0dfa8 --- /dev/null +++ b/gpAux/extensions/gps3ext/regress/input/1_14_thousands_of_files.source @@ -0,0 +1,17 @@ +CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS + '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; + +CREATE PROTOCOL s3 ( + readfunc = read_from_s3 +); + +DROP EXTERNAL TABLE s3regress; + +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/5120files/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; + +SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress; + +DROP EXTERNAL TABLE s3regress; + +DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/2_01_invalid_syntax.source b/gpAux/extensions/gps3ext/regress/input/2_01_invalid_syntax.source index 36911d240c..2f3cbe656b 100644 --- a/gpAux/extensions/gps3ext/regress/input/2_01_invalid_syntax.source +++ b/gpAux/extensions/gps3ext/regress/input/2_01_invalid_syntax.source @@ -1,18 +1,17 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date float, time float, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') format 'csv'; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (date float, time float, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/xac config=/home/gpadmin/s3.conf') format 'csv'; -DROP EXTERNAL TABLE s3example; +SELECT count(*) FROM s3regress; + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/2_02_invalid_region.source b/gpAux/extensions/gps3ext/regress/input/2_02_invalid_region.source index bde7009241..2ef1ca5252 100644 --- a/gpAux/extensions/gps3ext/regress/input/2_02_invalid_region.source +++ b/gpAux/extensions/gps3ext/regress/input/2_02_invalid_region.source @@ -1,18 +1,17 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://neverland.amazonaws.com/wherever/smalldata/ config=/home/gpadmin/s3.conf') format 'csv'; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://neverland.amazonaws.com/wherever/whatever/ config=/home/gpadmin/s3.conf') format 'csv'; -DROP EXTERNAL TABLE s3example; +SELECT count(*) FROM s3regress; + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/2_03_invalid_config.source b/gpAux/extensions/gps3ext/regress/input/2_03_invalid_config.source index ce41af75dc..389d9f348e 100644 --- a/gpAux/extensions/gps3ext/regress/input/2_03_invalid_config.source +++ b/gpAux/extensions/gps3ext/regress/input/2_03_invalid_config.source @@ -1,18 +1,17 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/') format 'csv'; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/') format 'csv'; -DROP EXTERNAL TABLE s3example; +SELECT count(*) FROM s3regress; + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/input/2_04_invalid_header.source b/gpAux/extensions/gps3ext/regress/input/2_04_invalid_header.source index ff73619ba2..d85d26524e 100644 --- a/gpAux/extensions/gps3ext/regress/input/2_04_invalid_header.source +++ b/gpAux/extensions/gps3ext/regress/input/2_04_invalid_header.source @@ -1,18 +1,17 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/csv_with_header/ config=/home/gpadmin/s3.conf') format 'csv'; +DROP EXTERNAL TABLE s3regress; -SELECT count(*) FROM s3example; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/csv_with_header/ config=/home/gpadmin/s3.conf') format 'csv'; -DROP EXTERNAL TABLE s3example; +SELECT count(*) FROM s3regress; + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_01_normal.source b/gpAux/extensions/gps3ext/regress/output/1_01_normal.source index da242a2570..4d27a9966c 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_01_normal.source +++ b/gpAux/extensions/gps3ext/regress/output/1_01_normal.source @@ -1,31 +1,17 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -SELECT count(*) FROM s3example; - count ----------- - 31033039 +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress; + count | sum | avg +----------+------------+----- + 31033039 | 1490754474 | 48 (1 row) -SELECT sum(open) FROM s3example; - sum ------------------- - 1490754474.37251 -(1 row) - -SELECT avg(open) FROM s3example; - avg ------------------- - 48.0376567171687 -(1 row) - -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_02_log_error.source b/gpAux/extensions/gps3ext/regress/output/1_02_log_error.source index 4cdb946e2d..36ed3e9718 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_02_log_error.source +++ b/gpAux/extensions/gps3ext/regress/output/1_02_log_error.source @@ -1,24 +1,22 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'text' ( DELIMITER ',' NULL '') LOG ERRORS SEGMENT REJECT LIMIT 5; -SELECT count(*) FROM s3example; +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'text' ( DELIMITER ',' NULL '') LOG ERRORS SEGMENT REJECT LIMIT 5; +SELECT count(*) FROM s3regress; count ---------- 31033039 (1 row) -SELECT gp_read_error_log('s3example'); +SELECT gp_read_error_log('s3regress'); gp_read_error_log ------------------- (0 rows) -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_03_bad_data.source b/gpAux/extensions/gps3ext/regress/output/1_03_bad_data.source index 32675073a1..581c6c5841 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_03_bad_data.source +++ b/gpAux/extensions/gps3ext/regress/output/1_03_bad_data.source @@ -1,15 +1,13 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; -SELECT count(*) FROM s3example; +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/baddata/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; +SELECT count(*) FROM s3regress; NOTICE: Found 3 data formatting errors (3 or more input rows). Rejected related input data. count ------- @@ -17,14 +15,14 @@ NOTICE: Found 3 data formatting errors (3 or more input rows). Rejected related (1 row) -- start_ignore -SELECT gp_read_error_log('s3example'); - gp_read_error_log ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - ("Mon Jul 04 20:11:08.360614 2016 PDT",s3example,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf",3,,"missing data for column ""low""","09/28/2009,09:30:06,35.49,35.37",) - ("Mon Jul 04 20:11:08.359313 2016 PDT",s3example,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf",2,,"missing data for column ""volume""","10/05/2009,36.19,36.18,36.2,100",) - ("Mon Jul 04 20:11:08.359313 2016 PDT",s3example,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf",4,,"missing data for column ""low""","10/05/2009,11:56:50,36.21,200",) +SELECT gp_read_error_log('s3regress'); + gp_read_error_log +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + ("Thu Jul 14 23:21:15.952755 2016 PDT",s3regress,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/baddata/ config=/home/gpadmin/s3.conf",3,,"missing data for column ""low""","09/28/2009,09:30:06,35.49,35.37",) + ("Thu Jul 14 23:21:15.953821 2016 PDT",s3regress,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/baddata/ config=/home/gpadmin/s3.conf",2,,"missing data for column ""volume""","10/05/2009,36.19,36.18,36.2,100",) + ("Thu Jul 14 23:21:15.953821 2016 PDT",s3regress,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/baddata/ config=/home/gpadmin/s3.conf",4,,"missing data for column ""low""","10/05/2009,11:56:50,36.21,200",) (3 rows) -- end_ignore -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_04_empty_prefix.source b/gpAux/extensions/gps3ext/regress/output/1_04_empty_prefix.source index f137391f38..253b440594 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_04_empty_prefix.source +++ b/gpAux/extensions/gps3ext/regress/output/1_04_empty_prefix.source @@ -1,24 +1,22 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/emptyfile/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; -SELECT count(*) FROM s3example; +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/emptyfile/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; +SELECT count(*) FROM s3regress; count ------- 0 (1 row) -SELECT gp_read_error_log('s3example'); +SELECT gp_read_error_log('s3regress'); gp_read_error_log ------------------- (0 rows) -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_05_one_line.source b/gpAux/extensions/gps3ext/regress/output/1_05_one_line.source index f9580d5f51..5009efde63 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_05_one_line.source +++ b/gpAux/extensions/gps3ext/regress/output/1_05_one_line.source @@ -1,19 +1,17 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/oneline/ config=/home/gpadmin/s3.conf') format 'csv'; -SELECT count(*) FROM s3example; +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/oneline/ config=/home/gpadmin/s3.conf') format 'csv'; +SELECT count(*) FROM s3regress; count ------- 1 (1 row) -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_06_1correct_1wrong.source b/gpAux/extensions/gps3ext/regress/output/1_06_1correct_1wrong.source index b6a88200e7..fdbfa79a0e 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_06_1correct_1wrong.source +++ b/gpAux/extensions/gps3ext/regress/output/1_06_1correct_1wrong.source @@ -1,15 +1,13 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/1correct1wrong/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; -SELECT count(*) FROM s3example; +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/1correct1wrong/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5; +SELECT count(*) FROM s3regress; NOTICE: Found 1 data formatting errors (1 or more input rows). Rejected related input data. count ------- @@ -17,12 +15,12 @@ NOTICE: Found 1 data formatting errors (1 or more input rows). Rejected related (1 row) -- start_ignore -SELECT gp_read_error_log('s3example'); - gp_read_error_log ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - ("Mon Jul 04 20:11:10.225613 2016 PDT",s3example,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/1correct1wrong/ config=/home/gpadmin/s3.conf",2,,"extra data after last expected column","whatever,09/28/2009,09:10:37,35.6,35.29,35.75,150,wherever",) +SELECT gp_read_error_log('s3regress'); + gp_read_error_log +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + ("Thu Jul 14 23:21:17.915597 2016 PDT",s3regress,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/1correct1wrong/ config=/home/gpadmin/s3.conf",2,,"extra data after last expected column","whatever,09/28/2009,09:10:37,35.6,35.29,35.75,150,wherever",) (1 row) -- end_ignore -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_07_huge_bad_data.source b/gpAux/extensions/gps3ext/regress/output/1_07_huge_bad_data.source index ab00a3ef7e..980438f044 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_07_huge_bad_data.source +++ b/gpAux/extensions/gps3ext/regress/output/1_07_huge_bad_data.source @@ -1,19 +1,17 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; -SELECT count(*) FROM s3example; +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/hugefile/airlinedata2.csv config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; +SELECT count(*) FROM s3regress; NOTICE: Found 2880938 data formatting errors (2880938 or more input rows). Rejected related input data. - count ------------ - 120654053 + count +---------- + 43983514 (1 row) -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_08_huge_correct_data.source b/gpAux/extensions/gps3ext/regress/output/1_08_huge_correct_data.source index cc612ed77e..bf95f905e6 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_08_huge_correct_data.source +++ b/gpAux/extensions/gps3ext/regress/output/1_08_huge_correct_data.source @@ -1,18 +1,16 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile_correct_data/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; -SELECT count(*) FROM s3example; - count ------------ - 124595613 +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/hugefile_correct_data/airlinedata1.csv config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; +SELECT count(*) FROM s3regress; + count +---------- + 47925074 (1 row) -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_09_partition.source b/gpAux/extensions/gps3ext/regress/output/1_09_partition.source index 2a4d05383c..91cb1d8246 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_09_partition.source +++ b/gpAux/extensions/gps3ext/regress/output/1_09_partition.source @@ -1,12 +1,10 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS -'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs + '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist CREATE TABLE stock (date text, time text, open float, high float, low float, volume int) DISTRIBUTED BY (date) PARTITION BY RANGE (volume) ( PARTITION stock10000 START (10000) INCLUSIVE, @@ -17,22 +15,22 @@ NOTICE: CREATE TABLE will create partition "stock_1_prt_stock10000" for table " NOTICE: CREATE TABLE will create partition "stock_1_prt_stock20000" for table "stock" NOTICE: CREATE TABLE will create partition "stock_1_prt_stock30000" for table "stock" INSERT INTO stock (date, time, open, high, low, volume) VALUES (03/08/2011, 094022, 35.25, 35.24, 35.29, 20001); -create READABLE external table ext_stock000 (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +CREATE READABLE EXTERNAL TABLE ext_stock000 (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/xao config=/home/gpadmin/s3.conf') FORMAT 'csv'; ALTER TABLE stock ADD PARTITION stock000 START (0) INCLUSIVE END (10000) EXCLUSIVE; NOTICE: CREATE TABLE will create partition "stock_1_prt_stock000" for table "stock" ALTER TABLE stock EXCHANGE PARTITION FOR (0) WITH TABLE ext_stock000 WITHOUT VALIDATION; NOTICE: exchanged partition "stock000" of relation "stock" with relation "ext_stock000" SELECT count(*) FROM stock_1_prt_stock000; - count ----------- - 31024410 + count +--------- + 1938855 (1 row) SELECT count(*) FROM stock; - count ----------- - 31024411 + count +--------- + 1938856 (1 row) DROP TABLE ext_stock000; diff --git a/gpAux/extensions/gps3ext/regress/output/1_10_all_regions.source b/gpAux/extensions/gps3ext/regress/output/1_10_all_regions.source index 2c1aadba9c..acf5a9afaa 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_10_all_regions.source +++ b/gpAux/extensions/gps3ext/regress/output/1_10_all_regions.source @@ -1,16 +1,14 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-east-1.amazonaws.com/us-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress + External table "public.s3regress" Column | Type | Modifiers --------+------------------+----------- date | text | @@ -23,19 +21,19 @@ Type: readable Encoding: UTF8 Format type: csv Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf" +External location: "s3://s3-us-east-1.amazonaws.com/us-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf" -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 +SELECT round(sum(open)) FROM s3regress; + round +--------- + 4239338 (1 row) -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-ap-northeast-2.amazonaws.com/apnortheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-1.amazonaws.com/us-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress + External table "public.s3regress" Column | Type | Modifiers --------+------------------+----------- date | text | @@ -48,19 +46,19 @@ Type: readable Encoding: UTF8 Format type: csv Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-ap-northeast-2.amazonaws.com/apnortheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf" +External location: "s3://s3-us-west-1.amazonaws.com/us-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf" -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 +SELECT round(sum(open)) FROM s3regress; + round +--------- + 4239338 (1 row) -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-ap-southeast-1.amazonaws.com/apsoutheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-south-1.amazonaws.com/ap-south-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress + External table "public.s3regress" Column | Type | Modifiers --------+------------------+----------- date | text | @@ -73,19 +71,19 @@ Type: readable Encoding: UTF8 Format type: csv Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-ap-southeast-1.amazonaws.com/apsoutheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf" +External location: "s3://s3-ap-south-1.amazonaws.com/ap-south-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf" -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 +SELECT round(sum(open)) FROM s3regress; + round +--------- + 4239338 (1 row) -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-ap-southeast-2.amazonaws.com/apsoutheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-northeast-2.amazonaws.com/ap-northeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress + External table "public.s3regress" Column | Type | Modifiers --------+------------------+----------- date | text | @@ -98,19 +96,19 @@ Type: readable Encoding: UTF8 Format type: csv Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-ap-southeast-2.amazonaws.com/apsoutheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf" +External location: "s3://s3-ap-northeast-2.amazonaws.com/ap-northeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf" -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 +SELECT round(sum(open)) FROM s3regress; + round +--------- + 4239338 (1 row) -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-southeast-1.amazonaws.com/ap-southeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress + External table "public.s3regress" Column | Type | Modifiers --------+------------------+----------- date | text | @@ -123,19 +121,19 @@ Type: readable Encoding: UTF8 Format type: csv Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf" +External location: "s3://s3-ap-southeast-1.amazonaws.com/ap-southeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf" -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 +SELECT round(sum(open)) FROM s3regress; + round +--------- + 4239338 (1 row) -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-eu-west-1.amazonaws.com/euwest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-southeast-2.amazonaws.com/ap-southeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress + External table "public.s3regress" Column | Type | Modifiers --------+------------------+----------- date | text | @@ -148,19 +146,19 @@ Type: readable Encoding: UTF8 Format type: csv Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-eu-west-1.amazonaws.com/euwest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf" +External location: "s3://s3-ap-southeast-2.amazonaws.com/ap-southeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf" -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 +SELECT round(sum(open)) FROM s3regress; + round +--------- + 4239338 (1 row) -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-northeast-1.amazonaws.com/ap-northeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress + External table "public.s3regress" Column | Type | Modifiers --------+------------------+----------- date | text | @@ -173,19 +171,19 @@ Type: readable Encoding: UTF8 Format type: csv Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf" +External location: "s3://s3-ap-northeast-1.amazonaws.com/ap-northeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf" -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 +SELECT round(sum(open)) FROM s3regress; + round +--------- + 4239338 (1 row) -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-east-1.amazonaws.com/useast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-eu-central-1.amazonaws.com/eu-central-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress + External table "public.s3regress" Column | Type | Modifiers --------+------------------+----------- date | text | @@ -198,19 +196,19 @@ Type: readable Encoding: UTF8 Format type: csv Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-us-east-1.amazonaws.com/useast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf" +External location: "s3://s3-eu-central-1.amazonaws.com/eu-central-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf" -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 +SELECT round(sum(open)) FROM s3regress; + round +--------- + 4239338 (1 row) -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-1.amazonaws.com/uswest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-eu-west-1.amazonaws.com/eu-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress + External table "public.s3regress" Column | Type | Modifiers --------+------------------+----------- date | text | @@ -223,13 +221,38 @@ Type: readable Encoding: UTF8 Format type: csv Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-us-west-1.amazonaws.com/uswest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf" +External location: "s3://s3-eu-west-1.amazonaws.com/eu-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf" -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 +SELECT round(sum(open)) FROM s3regress; + round +--------- + 4239338 (1 row) -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-sa-east-1.amazonaws.com/sa-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv'; +\d s3regress + External table "public.s3regress" + Column | Type | Modifiers +--------+------------------+----------- + date | text | + time | text | + open | double precision | + high | double precision | + low | double precision | + volume | integer | +Type: readable +Encoding: UTF8 +Format type: csv +Format options: delimiter ',' null '' escape '"' quote '"' +External location: "s3://s3-sa-east-1.amazonaws.com/sa-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf" + +SELECT round(sum(open)) FROM s3regress; + round +--------- + 4239338 +(1 row) + +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_11_gzipped_data.source b/gpAux/extensions/gps3ext/regress/output/1_11_gzipped_data.source index e4d9fb3d19..96164be0c1 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_11_gzipped_data.source +++ b/gpAux/extensions/gps3ext/regress/output/1_11_gzipped_data.source @@ -1,40 +1,26 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/gzipped_normal1/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -SELECT count(*) FROM s3example; - count ----------- - 31033039 -(1 row) - -SELECT sum(open) FROM s3example; - sum ------------------- - 1490754474.37203 -(1 row) - -SELECT avg(open) FROM s3example; - avg ------------------- - 48.0376567171532 +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_normal1/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress; + count | sum | avg +----------+------------+----- + 31033039 | 1490754474 | 48 (1 row) -drop external table s3example; -create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/gzipped_normal2/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; -SELECT count(*) FROM s3example; +DROP EXTERNAL TABLE s3regress; +CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_2002and2003/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT; +SELECT count(*) FROM s3regress; NOTICE: Found 335925 data formatting errors (335925 or more input rows). Rejected related input data. count ---------- - 52981760 + 11423976 (1 row) -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_12_no_prefix.source b/gpAux/extensions/gps3ext/regress/output/1_12_no_prefix.source index 5d959d3165..b1c2c6418a 100644 --- a/gpAux/extensions/gps3ext/regress/output/1_12_no_prefix.source +++ b/gpAux/extensions/gps3ext/regress/output/1_12_no_prefix.source @@ -1,85 +1,17 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" - Column | Type | Modifiers ---------+------------------+----------- - date | text | - time | text | - open | double precision | - high | double precision | - low | double precision | - volume | integer | -Type: readable -Encoding: UTF8 -Format type: csv -Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io config=/home/gpadmin/s3.conf" - -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 -(1 row) - -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" - Column | Type | Modifiers ---------+------------------+----------- - date | text | - time | text | - open | double precision | - high | double precision | - low | double precision | - volume | integer | -Type: readable -Encoding: UTF8 -Format type: csv -Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf" - -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 -(1 row) - -drop external table s3example; -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; -\d s3example - External table "public.s3example" - Column | Type | Modifiers ---------+------------------+----------- - date | text | - time | text | - open | double precision | - high | double precision | - low | double precision | - volume | integer | -Type: readable -Encoding: UTF8 -Format type: csv -Format options: delimiter ',' null '' escape '"' quote '"' -External location: "s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf" - -SELECT sum(open) FROM s3example; - sum ------------------- - 71676418.9189046 +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-ap-northeast-1.amazonaws.com/ap-northeast-1.s3test.pivotal.io config=/home/gpadmin/s3.conf') FORMAT 'csv'; +SELECT round(sum(open)) FROM s3regress; + round +---------- + 71676419 (1 row) -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/1_13_parallel1.source b/gpAux/extensions/gps3ext/regress/output/1_13_parallel1.source new file mode 100644 index 0000000000..a3c74fde4e --- /dev/null +++ b/gpAux/extensions/gps3ext/regress/output/1_13_parallel1.source @@ -0,0 +1,24 @@ +-- start_ignore +CREATE FUNCTION read_from_s3() RETURNS integer AS + '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; +ERROR: function "read_from_s3" already exists with same argument types +CREATE PROTOCOL s3 ( + readfunc = read_from_s3 +); +ERROR: duplicate key value violates unique constraint "pg_extprotocol_ptcname_index" +DETAIL: Key (ptcname)=(s3) already exists. +-- end_ignore +DROP EXTERNAL TABLE parallel1; +ERROR: table "parallel1" does not exist +CREATE READABLE EXTERNAL TABLE parallel1 (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM parallel1; + count | sum | avg +----------+------------+----- + 31033039 | 1490754474 | 48 +(1 row) + +DROP EXTERNAL TABLE parallel1; +-- start_ignore +DROP PROTOCOL s3; +-- end_ignore diff --git a/gpAux/extensions/gps3ext/regress/output/1_13_parallel2.source b/gpAux/extensions/gps3ext/regress/output/1_13_parallel2.source new file mode 100644 index 0000000000..e794996ecb --- /dev/null +++ b/gpAux/extensions/gps3ext/regress/output/1_13_parallel2.source @@ -0,0 +1,23 @@ +-- start_ignore +CREATE FUNCTION read_from_s3() RETURNS integer AS + '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; +ERROR: function "read_from_s3" already exists with same argument types +CREATE PROTOCOL s3 ( + readfunc = read_from_s3 +); +-- end_ignore +DROP EXTERNAL TABLE parallel2; +ERROR: table "parallel2" does not exist +CREATE READABLE EXTERNAL TABLE parallel2 (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM parallel2; + count | sum | avg +----------+------------+----- + 31033039 | 1490754474 | 48 +(1 row) + +DROP EXTERNAL TABLE parallel2; +-- start_ignore +DROP PROTOCOL s3; +ERROR: protocol "s3" does not exist +-- end_ignore diff --git a/gpAux/extensions/gps3ext/regress/output/1_14_thousands_of_files.source b/gpAux/extensions/gps3ext/regress/output/1_14_thousands_of_files.source new file mode 100644 index 0000000000..06037bd353 --- /dev/null +++ b/gpAux/extensions/gps3ext/regress/output/1_14_thousands_of_files.source @@ -0,0 +1,17 @@ +CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS + '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; +CREATE PROTOCOL s3 ( + readfunc = read_from_s3 +); +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/5120files/ config=/home/gpadmin/s3.conf') FORMAT 'csv'; +SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress; + count | sum | avg +--------+----------+----- + 655360 | 23302752 | 36 +(1 row) + +DROP EXTERNAL TABLE s3regress; +DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/2_01_invalid_syntax.source b/gpAux/extensions/gps3ext/regress/output/2_01_invalid_syntax.source index ebe09b1c4a..cc9247ac59 100644 --- a/gpAux/extensions/gps3ext/regress/output/2_01_invalid_syntax.source +++ b/gpAux/extensions/gps3ext/regress/output/2_01_invalid_syntax.source @@ -1,19 +1,14 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date float, time float, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') format 'csv'; -SELECT count(*) FROM s3example; -ERROR: invalid input syntax for type double precision: "09/28/2009" (seg0 slice1 ip-172-31-2-196.us-west-2.compute.internal:40000 pid=6425) -DETAIL: External table s3example, line 1 of s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf, column date -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date float, time float, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/xac config=/home/gpadmin/s3.conf') format 'csv'; +SELECT count(*) FROM s3regress; +ERROR: invalid input syntax for type double precision: "05/18/2010" (seg0 slice1 ip-172-31-2-196.us-west-2.compute.internal:40000 pid=23076) +DETAIL: External table s3regress, line 1 of s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/xac config=/home/gpadmin/s3.conf, column date +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; -WARNING: Releasing segworker groups since one or more segment connections failed. This will abort the transactions in the segments that did not get prepared. -NOTICE: Retry of the distributed transaction 'Abort Prepared' broadcast succeeded to the segments for gid = 1467688031-0000000206. -ERROR: The distributed transaction 'Prepare' broadcast failed to one or more segments for gid = 1467688031-0000000206. (cdbtm.c:689) diff --git a/gpAux/extensions/gps3ext/regress/output/2_02_invalid_region.source b/gpAux/extensions/gps3ext/regress/output/2_02_invalid_region.source index 041ef9ad05..e41009aafc 100644 --- a/gpAux/extensions/gps3ext/regress/output/2_02_invalid_region.source +++ b/gpAux/extensions/gps3ext/regress/output/2_02_invalid_region.source @@ -1,16 +1,14 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://neverland.amazonaws.com/wherever/smalldata/ config=/home/gpadmin/s3.conf') format 'csv'; -SELECT count(*) FROM s3example; -ERROR: Failed to init S3 extension, segid = 1, segnum = 6, please check your configurations and net connection: Failed to list bucket with retries: s3://neverland.amazonaws.com/wherever/smalldata/, Function: listBucketWithRetry, File: src/s3bucket_reader.cpp(139). (gps3ext.cpp:88) (seg1 slice1 ip-172-31-2-196.us-west-2.compute.internal:40001 pid=6518) (cdbdisp.c:1326) -DETAIL: External table s3example, file s3://neverland.amazonaws.com/wherever/smalldata/ config=/home/gpadmin/s3.conf -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://neverland.amazonaws.com/wherever/whatever/ config=/home/gpadmin/s3.conf') format 'csv'; +SELECT count(*) FROM s3regress; +ERROR: Failed to init S3 extension, segid = 5, segnum = 6, please check your configurations and net connection: Failed to list bucket for URL: s3://neverland.amazonaws.com/wherever/whatever/, Function: open, File: src/s3bucket_reader.cpp(55). (gps3ext.cpp:88) (seg5 slice1 ip-172-31-2-195.us-west-2.compute.internal:40001 pid=2338) (cdbdisp.c:1326) +DETAIL: External table s3regress, file s3://neverland.amazonaws.com/wherever/whatever/ config=/home/gpadmin/s3.conf +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/2_03_invalid_config.source b/gpAux/extensions/gps3ext/regress/output/2_03_invalid_config.source index 1c3c3c8984..ea33f9d180 100644 --- a/gpAux/extensions/gps3ext/regress/output/2_03_invalid_config.source +++ b/gpAux/extensions/gps3ext/regress/output/2_03_invalid_config.source @@ -1,16 +1,14 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/') format 'csv'; -SELECT count(*) FROM s3example; -ERROR: Failed to init S3 extension, segid = -1, segnum = -1, please check your configurations and net connection: (gps3ext.cpp:88) (seg0 slice1 ip-172-31-2-196.us-west-2.compute.internal:40000 pid=6560) (cdbdisp.c:1326) -DETAIL: External table s3example, file s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/') format 'csv'; +SELECT count(*) FROM s3regress; +ERROR: Failed to init S3 extension, segid = -1, segnum = -1, please check your configurations and net connection: (gps3ext.cpp:88) (seg2 slice1 ip-172-31-2-194.us-west-2.compute.internal:40000 pid=2380) (cdbdisp.c:1326) +DETAIL: External table s3regress, file s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/output/2_04_invalid_header.source b/gpAux/extensions/gps3ext/regress/output/2_04_invalid_header.source index 847afc36ba..6c8683d514 100644 --- a/gpAux/extensions/gps3ext/regress/output/2_04_invalid_header.source +++ b/gpAux/extensions/gps3ext/regress/output/2_04_invalid_header.source @@ -1,16 +1,14 @@ --- create the database functions CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE; --- declare the protocol name along with in/out funcs CREATE PROTOCOL s3 ( readfunc = read_from_s3 ); -drop external table s3example; -ERROR: table "s3example" does not exist -create READABLE external table s3example (date text, time text, open float, high float, - low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/csv_with_header/ config=/home/gpadmin/s3.conf') format 'csv'; -SELECT count(*) FROM s3example; -ERROR: invalid input syntax for type double precision: "open" (seg0 slice1 ip-172-31-2-196.us-west-2.compute.internal:40000 pid=6607) -DETAIL: External table s3example, line 1 of s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/csv_with_header/ config=/home/gpadmin/s3.conf, column open -DROP EXTERNAL TABLE s3example; +DROP EXTERNAL TABLE s3regress; +ERROR: table "s3regress" does not exist +CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float, + low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/csv_with_header/ config=/home/gpadmin/s3.conf') format 'csv'; +SELECT count(*) FROM s3regress; +ERROR: invalid input syntax for type double precision: "open" (seg0 slice1 ip-172-31-2-196.us-west-2.compute.internal:40000 pid=23220) +DETAIL: External table s3regress, line 1 of s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/csv_with_header/ config=/home/gpadmin/s3.conf, column open +DROP EXTERNAL TABLE s3regress; DROP PROTOCOL s3; diff --git a/gpAux/extensions/gps3ext/regress/s3_regress_schedule b/gpAux/extensions/gps3ext/regress/s3_regress_schedule index 8063e41988..e9fbb423f1 100644 --- a/gpAux/extensions/gps3ext/regress/s3_regress_schedule +++ b/gpAux/extensions/gps3ext/regress/s3_regress_schedule @@ -10,6 +10,8 @@ test: 1_09_partition test: 1_10_all_regions test: 1_11_gzipped_data test: 1_12_no_prefix +test: 1_13_parallel1 1_13_parallel2 +test: 1_14_thousands_of_files test: 2_01_invalid_syntax test: 2_02_invalid_region test: 2_03_invalid_config -- GitLab