Skip to content

G
Gpdb

Greenplum / Gpdb

通知 7
Star 1
Fork 0
G
Gpdb

体验新版 GitCode，发现更多精彩内容 >>

提交 712705e1 编写于 作者: A Adam Lee 提交者： GitHub
浏览文件
操作

s3ext: update regress cases (#950) 

1. speed up running by specifying proper files with sufficient size
2. add a pair of tests which connect to the same S3 file in parallel
3. revise sql scripts to make them neat
Signed-off-by: NAdam Lee <ali@pivotal.io>
Signed-off-by: NKuien Liu <kliu@pivotal.io>
上级 9625f147
隐藏空白更改
内联 并排
Showing with 548 addition and 545 deletion
gpAux/extensions/gps3ext/regress/gpcheckcloud_regress.sh
浏览文件 @ 712705e1
......@@ -3,16 +3,16 @@
EXIT_CODE=0
GPCHECKCLOUD=./bin/gpcheckcloud/gpcheckcloud
PREFIX[0]="http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0014"
PREFIX[1]="http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0016"
PREFIX[2]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0014"
PREFIX[3]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0016"
PREFIX[4]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/gzipped/data0001.gz"
PREFIX[5]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/threebytes/"
PREFIX[6]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/"
PREFIX[7]="s3://s3-us-east-1.amazonaws.com/useast1.s3test.pivotal.io/small17/"
PREFIX[8]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/gzipped/"
PREFIX[9]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/gzipped_normal1/"
PREFIX[0]="http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/small17/data0014"
PREFIX[1]="http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/small17/data0016"
PREFIX[2]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/small17/data0014"
PREFIX[3]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/small17/data0016"
PREFIX[4]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped/data0001.gz"
PREFIX[5]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/threebytes/"
PREFIX[6]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/small17/"
PREFIX[7]="s3://s3-us-east-1.amazonaws.com/us-east-1.s3test.pivotal.io/regress/small17/"
PREFIX[8]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped/"
PREFIX[9]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_normal1/"
HASHSUM[0]="68c4a63b721e7af0ae945ce109ca87ad"
HASHSUM[1]="0fd502a303eb8f138f5916ec357721b1"
......@@ -26,13 +26,13 @@ HASHSUM[8]="7b2260e9a3a3f26e84aa28dc2124f68f"
HASHSUM[9]="eacb7b210d3f7703ee06d16f520b103e"
#HUGE FILES
PREFIX[10]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/airlinedata1.csv"
PREFIX[11]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/"
PREFIX[12]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/gzipped_normal2/"
PREFIX[10]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/hugefile/airlinedata1.csv"
PREFIX[11]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_normal2/"
PREFIX[12]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/5120files/"
HASHSUM[10]="f5811ad92c994f1d6913d5338575fe38"
HASHSUM[11]="75baaa39f2b1544ed8af437c2cad86b7"
HASHSUM[12]="a930794bc885bccf6eed45bd40367a7d"
HASHSUM[11]="a930794bc885bccf6eed45bd40367a7d"
HASHSUM[12]="71277a7004842241191cc65146be24f6"
for ((i=0; i<${#PREFIX[@]}; i++))
do
......
gpAux/extensions/gps3ext/regress/input/1_01_normal.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
SELECT sum(open) FROM s3example;
SELECT avg(open) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
DROP EXTERNAL TABLE s3example;
SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/1_02_log_error.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'text' ( DELIMITER ',' NULL '') LOG ERRORS SEGMENT REJECT LIMIT 5;
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
SELECT gp_read_error_log('s3example');
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'text' ( DELIMITER ',' NULL '') LOG ERRORS SEGMENT REJECT LIMIT 5;
DROP EXTERNAL TABLE s3example;
SELECT count(*) FROM s3regress;
SELECT gp_read_error_log('s3regress');
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/1_03_bad_data.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/baddata/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
SELECT count(*) FROM s3regress;
-- start_ignore
SELECT gp_read_error_log('s3example');
SELECT gp_read_error_log('s3regress');
-- end_ignore
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/1_04_empty_prefix.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/emptyfile/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
SELECT gp_read_error_log('s3example');
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/emptyfile/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
DROP EXTERNAL TABLE s3example;
SELECT count(*) FROM s3regress;
SELECT gp_read_error_log('s3regress');
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/1_05_one_line.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/oneline/ config=/home/gpadmin/s3.conf') format 'csv';
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/oneline/ config=/home/gpadmin/s3.conf') format 'csv';
DROP EXTERNAL TABLE s3example;
SELECT count(*) FROM s3regress;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/1_06_1correct_1wrong.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/1correct1wrong/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/1correct1wrong/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
SELECT count(*) FROM s3regress;
SELECT count(*) FROM s3example;
-- start_ignore
SELECT gp_read_error_log('s3example');
SELECT gp_read_error_log('s3regress');
-- end_ignore
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/1_07_huge_bad_data.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/hugefile/airlinedata2.csv config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
DROP EXTERNAL TABLE s3example;
SELECT count(*) FROM s3regress;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/1_08_huge_correct_data.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile_correct_data/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/hugefile_correct_data/airlinedata1.csv config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
DROP EXTERNAL TABLE s3example;
SELECT count(*) FROM s3regress;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/1_09_partition.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
DROP EXTERNAL TABLE s3regress;
CREATE TABLE stock (date text, time text, open float, high float, low float, volume int) DISTRIBUTED BY (date) PARTITION BY RANGE (volume)
(
......@@ -18,8 +16,8 @@ CREATE TABLE stock (date text, time text, open float, high float, low float, vol
INSERT INTO stock (date, time, open, high, low, volume) VALUES (03/08/2011, 094022, 35.25, 35.24, 35.29, 20001);
create READABLE external table ext_stock000 (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
CREATE READABLE EXTERNAL TABLE ext_stock000 (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/xao config=/home/gpadmin/s3.conf') FORMAT 'csv';
ALTER TABLE stock ADD PARTITION stock000 START (0) INCLUSIVE END (10000) EXCLUSIVE;
......
gpAux/extensions/gps3ext/regress/input/1_10_all_regions.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
DROP EXTERNAL TABLE s3regress;
\d s3example
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-east-1.amazonaws.com/us-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT sum(open) FROM s3example;
\d s3regress
SELECT round(sum(open)) FROM s3regress;
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-ap-northeast-2.amazonaws.com/apnortheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
DROP EXTERNAL TABLE s3regress;
SELECT sum(open) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-1.amazonaws.com/us-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-ap-southeast-1.amazonaws.com/apsoutheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT round(sum(open)) FROM s3regress;
\d s3example
SELECT sum(open) FROM s3example;
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-south-1.amazonaws.com/ap-south-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-ap-southeast-2.amazonaws.com/apsoutheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
\d s3example
SELECT round(sum(open)) FROM s3regress;
SELECT sum(open) FROM s3example;
DROP EXTERNAL TABLE s3regress;
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-northeast-2.amazonaws.com/ap-northeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
\d s3regress
SELECT sum(open) FROM s3example;
SELECT round(sum(open)) FROM s3regress;
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-eu-west-1.amazonaws.com/euwest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
DROP EXTERNAL TABLE s3regress;
\d s3example
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-southeast-1.amazonaws.com/ap-southeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT sum(open) FROM s3example;
\d s3regress
SELECT round(sum(open)) FROM s3regress;
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
DROP EXTERNAL TABLE s3regress;
SELECT sum(open) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-southeast-2.amazonaws.com/ap-southeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-east-1.amazonaws.com/useast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT round(sum(open)) FROM s3regress;
\d s3example
SELECT sum(open) FROM s3example;
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-northeast-1.amazonaws.com/ap-northeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-1.amazonaws.com/uswest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
\d s3example
SELECT round(sum(open)) FROM s3regress;
SELECT sum(open) FROM s3example;
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-eu-central-1.amazonaws.com/eu-central-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
SELECT round(sum(open)) FROM s3regress;
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-eu-west-1.amazonaws.com/eu-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
SELECT round(sum(open)) FROM s3regress;
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-sa-east-1.amazonaws.com/sa-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
SELECT round(sum(open)) FROM s3regress;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/1_11_gzipped_data.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/gzipped_normal1/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
SELECT sum(open) FROM s3example;
SELECT avg(open) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_normal1/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
drop external table s3example;
create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/gzipped_normal2/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress;
SELECT count(*) FROM s3example;
DROP EXTERNAL TABLE s3regress;
DROP EXTERNAL TABLE s3example;
CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_2002and2003/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
SELECT count(*) FROM s3regress;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/1_12_no_prefix.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
DROP EXTERNAL TABLE s3regress;
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io config=/home/gpadmin/s3.conf') FORMAT 'csv';
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-northeast-1.amazonaws.com/ap-northeast-1.s3test.pivotal.io config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
SELECT round(sum(open)) FROM s3regress;
SELECT sum(open) FROM s3example;
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
SELECT sum(open) FROM s3example;
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
SELECT sum(open) FROM s3example;
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/1_13_parallel1.source 0 → 100644
浏览文件 @ 712705e1
-- start_ignore
CREATE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
-- end_ignore
DROP EXTERNAL TABLE parallel1;
CREATE READABLE EXTERNAL TABLE parallel1 (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM parallel1;
DROP EXTERNAL TABLE parallel1;
-- start_ignore
DROP PROTOCOL s3;
-- end_ignore
gpAux/extensions/gps3ext/regress/input/1_13_parallel2.source 0 → 100644
浏览文件 @ 712705e1
-- start_ignore
CREATE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
-- end_ignore
DROP EXTERNAL TABLE parallel2;
CREATE READABLE EXTERNAL TABLE parallel2 (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM parallel2;
DROP EXTERNAL TABLE parallel2;
-- start_ignore
DROP PROTOCOL s3;
-- end_ignore
gpAux/extensions/gps3ext/regress/input/1_14_thousands_of_files.source 0 → 100644
浏览文件 @ 712705e1
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/5120files/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/2_01_invalid_syntax.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date float, time float, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') format 'csv';
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (date float, time float, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/xac config=/home/gpadmin/s3.conf') format 'csv';
DROP EXTERNAL TABLE s3example;
SELECT count(*) FROM s3regress;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/2_02_invalid_region.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://neverland.amazonaws.com/wherever/smalldata/ config=/home/gpadmin/s3.conf') format 'csv';
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://neverland.amazonaws.com/wherever/whatever/ config=/home/gpadmin/s3.conf') format 'csv';
DROP EXTERNAL TABLE s3example;
SELECT count(*) FROM s3regress;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/2_03_invalid_config.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/') format 'csv';
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/') format 'csv';
DROP EXTERNAL TABLE s3example;
SELECT count(*) FROM s3regress;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/input/2_04_invalid_header.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/csv_with_header/ config=/home/gpadmin/s3.conf') format 'csv';
DROP EXTERNAL TABLE s3regress;
SELECT count(*) FROM s3example;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/csv_with_header/ config=/home/gpadmin/s3.conf') format 'csv';
DROP EXTERNAL TABLE s3example;
SELECT count(*) FROM s3regress;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_01_normal.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT count(*) FROM s3example;
count
----------
31033039
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress;
count | sum | avg
----------+------------+-----
31033039 | 1490754474 | 48
(1 row)
SELECT sum(open) FROM s3example;
sum
------------------
1490754474.37251
(1 row)
SELECT avg(open) FROM s3example;
avg
------------------
48.0376567171687
(1 row)
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_02_log_error.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'text' ( DELIMITER ',' NULL '') LOG ERRORS SEGMENT REJECT LIMIT 5;
SELECT count(*) FROM s3example;
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'text' ( DELIMITER ',' NULL '') LOG ERRORS SEGMENT REJECT LIMIT 5;
SELECT count(*) FROM s3regress;
count
----------
31033039
(1 row)
SELECT gp_read_error_log('s3example');
SELECT gp_read_error_log('s3regress');
gp_read_error_log
-------------------
(0 rows)
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_03_bad_data.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
SELECT count(*) FROM s3example;
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/baddata/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
SELECT count(*) FROM s3regress;
NOTICE: Found 3 data formatting errors (3 or more input rows). Rejected related input data.
count
-------
......@@ -17,14 +15,14 @@ NOTICE: Found 3 data formatting errors (3 or more input rows). Rejected related
(1 row)
-- start_ignore
SELECT gp_read_error_log('s3example');
gp_read_error_log
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
("Mon Jul 04 20:11:08.360614 2016 PDT",s3example,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf",3,,"missing data for column ""low""","09/28/2009,09:30:06,35.49,35.37",)
("Mon Jul 04 20:11:08.359313 2016 PDT",s3example,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf",2,,"missing data for column ""volume""","10/05/2009,36.19,36.18,36.2,100",)
("Mon Jul 04 20:11:08.359313 2016 PDT",s3example,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf",4,,"missing data for column ""low""","10/05/2009,11:56:50,36.21,200",)
SELECT gp_read_error_log('s3regress');
gp_read_error_log
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
("Thu Jul 14 23:21:15.952755 2016 PDT",s3regress,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/baddata/ config=/home/gpadmin/s3.conf",3,,"missing data for column ""low""","09/28/2009,09:30:06,35.49,35.37",)
("Thu Jul 14 23:21:15.953821 2016 PDT",s3regress,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/baddata/ config=/home/gpadmin/s3.conf",2,,"missing data for column ""volume""","10/05/2009,36.19,36.18,36.2,100",)
("Thu Jul 14 23:21:15.953821 2016 PDT",s3regress,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/baddata/ config=/home/gpadmin/s3.conf",4,,"missing data for column ""low""","10/05/2009,11:56:50,36.21,200",)
(3 rows)
-- end_ignore
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_04_empty_prefix.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/emptyfile/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
SELECT count(*) FROM s3example;
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/emptyfile/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
SELECT count(*) FROM s3regress;
count
-------
0
(1 row)
SELECT gp_read_error_log('s3example');
SELECT gp_read_error_log('s3regress');
gp_read_error_log
-------------------
(0 rows)
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_05_one_line.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/oneline/ config=/home/gpadmin/s3.conf') format 'csv';
SELECT count(*) FROM s3example;
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/oneline/ config=/home/gpadmin/s3.conf') format 'csv';
SELECT count(*) FROM s3regress;
count
-------
1
(1 row)
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_06_1correct_1wrong.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/1correct1wrong/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
SELECT count(*) FROM s3example;
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/1correct1wrong/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
SELECT count(*) FROM s3regress;
NOTICE: Found 1 data formatting errors (1 or more input rows). Rejected related input data.
count
-------
......@@ -17,12 +15,12 @@ NOTICE: Found 1 data formatting errors (1 or more input rows). Rejected related
(1 row)
-- start_ignore
SELECT gp_read_error_log('s3example');
gp_read_error_log
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
("Mon Jul 04 20:11:10.225613 2016 PDT",s3example,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/1correct1wrong/ config=/home/gpadmin/s3.conf",2,,"extra data after last expected column","whatever,09/28/2009,09:10:37,35.6,35.29,35.75,150,wherever",)
SELECT gp_read_error_log('s3regress');
gp_read_error_log
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
("Thu Jul 14 23:21:17.915597 2016 PDT",s3regress,"s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/1correct1wrong/ config=/home/gpadmin/s3.conf",2,,"extra data after last expected column","whatever,09/28/2009,09:10:37,35.6,35.29,35.75,150,wherever",)
(1 row)
-- end_ignore
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_07_huge_bad_data.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
SELECT count(*) FROM s3example;
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/hugefile/airlinedata2.csv config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
SELECT count(*) FROM s3regress;
NOTICE: Found 2880938 data formatting errors (2880938 or more input rows). Rejected related input data.
count
-----------
120654053
count
----------
43983514
(1 row)
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_08_huge_correct_data.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile_correct_data/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
SELECT count(*) FROM s3example;
count
-----------
124595613
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/hugefile_correct_data/airlinedata1.csv config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
SELECT count(*) FROM s3regress;
count
----------
47925074
(1 row)
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_09_partition.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE TABLE stock (date text, time text, open float, high float, low float, volume int) DISTRIBUTED BY (date) PARTITION BY RANGE (volume)
(
PARTITION stock10000 START (10000) INCLUSIVE,
......@@ -17,22 +15,22 @@ NOTICE: CREATE TABLE will create partition "stock_1_prt_stock10000" for table "
NOTICE: CREATE TABLE will create partition "stock_1_prt_stock20000" for table "stock"
NOTICE: CREATE TABLE will create partition "stock_1_prt_stock30000" for table "stock"
INSERT INTO stock (date, time, open, high, low, volume) VALUES (03/08/2011, 094022, 35.25, 35.24, 35.29, 20001);
create READABLE external table ext_stock000 (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
CREATE READABLE EXTERNAL TABLE ext_stock000 (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/xao config=/home/gpadmin/s3.conf') FORMAT 'csv';
ALTER TABLE stock ADD PARTITION stock000 START (0) INCLUSIVE END (10000) EXCLUSIVE;
NOTICE: CREATE TABLE will create partition "stock_1_prt_stock000" for table "stock"
ALTER TABLE stock EXCHANGE PARTITION FOR (0) WITH TABLE ext_stock000 WITHOUT VALIDATION;
NOTICE: exchanged partition "stock000" of relation "stock" with relation "ext_stock000"
SELECT count(*) FROM stock_1_prt_stock000;
count
----------
31024410
count
---------
1938855
(1 row)
SELECT count(*) FROM stock;
count
----------
31024411
count
---------
1938856
(1 row)
DROP TABLE ext_stock000;
......
gpAux/extensions/gps3ext/regress/output/1_10_all_regions.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-east-1.amazonaws.com/us-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
External table "public.s3regress"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
......@@ -23,19 +21,19 @@ Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf"
External location: "s3://s3-us-east-1.amazonaws.com/us-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
SELECT round(sum(open)) FROM s3regress;
round
---------
4239338
(1 row)
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-ap-northeast-2.amazonaws.com/apnortheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-1.amazonaws.com/us-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
External table "public.s3regress"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
......@@ -48,19 +46,19 @@ Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-ap-northeast-2.amazonaws.com/apnortheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf"
External location: "s3://s3-us-west-1.amazonaws.com/us-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
SELECT round(sum(open)) FROM s3regress;
round
---------
4239338
(1 row)
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-ap-southeast-1.amazonaws.com/apsoutheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-south-1.amazonaws.com/ap-south-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
External table "public.s3regress"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
......@@ -73,19 +71,19 @@ Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-ap-southeast-1.amazonaws.com/apsoutheast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf"
External location: "s3://s3-ap-south-1.amazonaws.com/ap-south-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
SELECT round(sum(open)) FROM s3regress;
round
---------
4239338
(1 row)
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-ap-southeast-2.amazonaws.com/apsoutheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-northeast-2.amazonaws.com/ap-northeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
External table "public.s3regress"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
......@@ -98,19 +96,19 @@ Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-ap-southeast-2.amazonaws.com/apsoutheast2.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf"
External location: "s3://s3-ap-northeast-2.amazonaws.com/ap-northeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
SELECT round(sum(open)) FROM s3regress;
round
---------
4239338
(1 row)
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-southeast-1.amazonaws.com/ap-southeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
External table "public.s3regress"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
......@@ -123,19 +121,19 @@ Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf"
External location: "s3://s3-ap-southeast-1.amazonaws.com/ap-southeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
SELECT round(sum(open)) FROM s3regress;
round
---------
4239338
(1 row)
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-eu-west-1.amazonaws.com/euwest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-southeast-2.amazonaws.com/ap-southeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
External table "public.s3regress"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
......@@ -148,19 +146,19 @@ Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-eu-west-1.amazonaws.com/euwest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf"
External location: "s3://s3-ap-southeast-2.amazonaws.com/ap-southeast-2.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
SELECT round(sum(open)) FROM s3regress;
round
---------
4239338
(1 row)
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-northeast-1.amazonaws.com/ap-northeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
External table "public.s3regress"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
......@@ -173,19 +171,19 @@ Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf"
External location: "s3://s3-ap-northeast-1.amazonaws.com/ap-northeast-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
SELECT round(sum(open)) FROM s3regress;
round
---------
4239338
(1 row)
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-east-1.amazonaws.com/useast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-eu-central-1.amazonaws.com/eu-central-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
External table "public.s3regress"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
......@@ -198,19 +196,19 @@ Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-us-east-1.amazonaws.com/useast1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf"
External location: "s3://s3-eu-central-1.amazonaws.com/eu-central-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
SELECT round(sum(open)) FROM s3regress;
round
---------
4239338
(1 row)
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-1.amazonaws.com/uswest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-eu-west-1.amazonaws.com/eu-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
External table "public.s3regress"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
......@@ -223,13 +221,38 @@ Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-us-west-1.amazonaws.com/uswest1.s3test.pivotal.io/small17/ config=/home/gpadmin/s3.conf"
External location: "s3://s3-eu-west-1.amazonaws.com/eu-west-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
SELECT round(sum(open)) FROM s3regress;
round
---------
4239338
(1 row)
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-sa-east-1.amazonaws.com/sa-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3regress
External table "public.s3regress"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
time | text |
open | double precision |
high | double precision |
low | double precision |
volume | integer |
Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-sa-east-1.amazonaws.com/sa-east-1.s3test.pivotal.io/regress/small17/data0000 config=/home/gpadmin/s3.conf"
SELECT round(sum(open)) FROM s3regress;
round
---------
4239338
(1 row)
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_11_gzipped_data.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/gzipped_normal1/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT count(*) FROM s3example;
count
----------
31033039
(1 row)
SELECT sum(open) FROM s3example;
sum
------------------
1490754474.37203
(1 row)
SELECT avg(open) FROM s3example;
avg
------------------
48.0376567171532
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_normal1/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress;
count | sum | avg
----------+------------+-----
31033039 | 1490754474 | 48
(1 row)
drop external table s3example;
create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/gzipped_normal2/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
SELECT count(*) FROM s3example;
DROP EXTERNAL TABLE s3regress;
CREATE READABLE EXTERNAL TABLE s3regress (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/gzipped_2002and2003/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
SELECT count(*) FROM s3regress;
NOTICE: Found 335925 data formatting errors (335925 or more input rows). Rejected related input data.
count
----------
52981760
11423976
(1 row)
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_12_no_prefix.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
time | text |
open | double precision |
high | double precision |
low | double precision |
volume | integer |
Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-ap-northeast-1.amazonaws.com/apnortheast1.s3test.pivotal.io config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
(1 row)
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
time | text |
open | double precision |
high | double precision |
low | double precision |
volume | integer |
Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-eu-central-1.amazonaws.com/eucentral1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
(1 row)
drop external table s3example;
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
\d s3example
External table "public.s3example"
Column | Type | Modifiers
--------+------------------+-----------
date | text |
time | text |
open | double precision |
high | double precision |
low | double precision |
volume | integer |
Type: readable
Encoding: UTF8
Format type: csv
Format options: delimiter ',' null '' escape '"' quote '"'
External location: "s3://s3-sa-east-1.amazonaws.com/saeast1.s3test.pivotal.io/ config=/home/gpadmin/s3.conf"
SELECT sum(open) FROM s3example;
sum
------------------
71676418.9189046
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-ap-northeast-1.amazonaws.com/ap-northeast-1.s3test.pivotal.io config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT round(sum(open)) FROM s3regress;
round
----------
71676419
(1 row)
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/1_13_parallel1.source 0 → 100644
浏览文件 @ 712705e1
-- start_ignore
CREATE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
ERROR: function "read_from_s3" already exists with same argument types
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
ERROR: duplicate key value violates unique constraint "pg_extprotocol_ptcname_index"
DETAIL: Key (ptcname)=(s3) already exists.
-- end_ignore
DROP EXTERNAL TABLE parallel1;
ERROR: table "parallel1" does not exist
CREATE READABLE EXTERNAL TABLE parallel1 (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM parallel1;
count | sum | avg
----------+------------+-----
31033039 | 1490754474 | 48
(1 row)
DROP EXTERNAL TABLE parallel1;
-- start_ignore
DROP PROTOCOL s3;
-- end_ignore
gpAux/extensions/gps3ext/regress/output/1_13_parallel2.source 0 → 100644
浏览文件 @ 712705e1
-- start_ignore
CREATE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
ERROR: function "read_from_s3" already exists with same argument types
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
-- end_ignore
DROP EXTERNAL TABLE parallel2;
ERROR: table "parallel2" does not exist
CREATE READABLE EXTERNAL TABLE parallel2 (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM parallel2;
count | sum | avg
----------+------------+-----
31033039 | 1490754474 | 48
(1 row)
DROP EXTERNAL TABLE parallel2;
-- start_ignore
DROP PROTOCOL s3;
ERROR: protocol "s3" does not exist
-- end_ignore
gpAux/extensions/gps3ext/regress/output/1_14_thousands_of_files.source 0 → 100644
浏览文件 @ 712705e1
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/5120files/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
SELECT count(*) count, round(sum(open)) sum, round(avg(open)) avg FROM s3regress;
count | sum | avg
--------+----------+-----
655360 | 23302752 | 36
(1 row)
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/2_01_invalid_syntax.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date float, time float, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') format 'csv';
SELECT count(*) FROM s3example;
ERROR: invalid input syntax for type double precision: "09/28/2009" (seg0 slice1 ip-172-31-2-196.us-west-2.compute.internal:40000 pid=6425)
DETAIL: External table s3example, line 1 of s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf, column date
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date float, time float, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/xac config=/home/gpadmin/s3.conf') format 'csv';
SELECT count(*) FROM s3regress;
ERROR: invalid input syntax for type double precision: "05/18/2010" (seg0 slice1 ip-172-31-2-196.us-west-2.compute.internal:40000 pid=23076)
DETAIL: External table s3regress, line 1 of s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/xac config=/home/gpadmin/s3.conf, column date
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
WARNING: Releasing segworker groups since one or more segment connections failed. This will abort the transactions in the segments that did not get prepared.
NOTICE: Retry of the distributed transaction 'Abort Prepared' broadcast succeeded to the segments for gid = 1467688031-0000000206.
ERROR: The distributed transaction 'Prepare' broadcast failed to one or more segments for gid = 1467688031-0000000206. (cdbtm.c:689)
gpAux/extensions/gps3ext/regress/output/2_02_invalid_region.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://neverland.amazonaws.com/wherever/smalldata/ config=/home/gpadmin/s3.conf') format 'csv';
SELECT count(*) FROM s3example;
ERROR: Failed to init S3 extension, segid = 1, segnum = 6, please check your configurations and net connection: Failed to list bucket with retries: s3://neverland.amazonaws.com/wherever/smalldata/, Function: listBucketWithRetry, File: src/s3bucket_reader.cpp(139). (gps3ext.cpp:88) (seg1 slice1 ip-172-31-2-196.us-west-2.compute.internal:40001 pid=6518) (cdbdisp.c:1326)
DETAIL: External table s3example, file s3://neverland.amazonaws.com/wherever/smalldata/ config=/home/gpadmin/s3.conf
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://neverland.amazonaws.com/wherever/whatever/ config=/home/gpadmin/s3.conf') format 'csv';
SELECT count(*) FROM s3regress;
ERROR: Failed to init S3 extension, segid = 5, segnum = 6, please check your configurations and net connection: Failed to list bucket for URL: s3://neverland.amazonaws.com/wherever/whatever/, Function: open, File: src/s3bucket_reader.cpp(55). (gps3ext.cpp:88) (seg5 slice1 ip-172-31-2-195.us-west-2.compute.internal:40001 pid=2338) (cdbdisp.c:1326)
DETAIL: External table s3regress, file s3://neverland.amazonaws.com/wherever/whatever/ config=/home/gpadmin/s3.conf
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/2_03_invalid_config.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/') format 'csv';
SELECT count(*) FROM s3example;
ERROR: Failed to init S3 extension, segid = -1, segnum = -1, please check your configurations and net connection: (gps3ext.cpp:88) (seg0 slice1 ip-172-31-2-196.us-west-2.compute.internal:40000 pid=6560) (cdbdisp.c:1326)
DETAIL: External table s3example, file s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/') format 'csv';
SELECT count(*) FROM s3regress;
ERROR: Failed to init S3 extension, segid = -1, segnum = -1, please check your configurations and net connection: (gps3ext.cpp:88) (seg2 slice1 ip-172-31-2-194.us-west-2.compute.internal:40000 pid=2380) (cdbdisp.c:1326)
DETAIL: External table s3regress, file s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/normal/
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/output/2_04_invalid_header.source
浏览文件 @ 712705e1
-- create the database functions
CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
'$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
-- declare the protocol name along with in/out funcs
CREATE PROTOCOL s3 (
readfunc = read_from_s3
);
drop external table s3example;
ERROR: table "s3example" does not exist
create READABLE external table s3example (date text, time text, open float, high float,
low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/csv_with_header/ config=/home/gpadmin/s3.conf') format 'csv';
SELECT count(*) FROM s3example;
ERROR: invalid input syntax for type double precision: "open" (seg0 slice1 ip-172-31-2-196.us-west-2.compute.internal:40000 pid=6607)
DETAIL: External table s3example, line 1 of s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/csv_with_header/ config=/home/gpadmin/s3.conf, column open
DROP EXTERNAL TABLE s3example;
DROP EXTERNAL TABLE s3regress;
ERROR: table "s3regress" does not exist
CREATE READABLE EXTERNAL TABLE s3regress (date text, time text, open float, high float,
low float, volume int) LOCATION('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/csv_with_header/ config=/home/gpadmin/s3.conf') format 'csv';
SELECT count(*) FROM s3regress;
ERROR: invalid input syntax for type double precision: "open" (seg0 slice1 ip-172-31-2-196.us-west-2.compute.internal:40000 pid=23220)
DETAIL: External table s3regress, line 1 of s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/regress/csv_with_header/ config=/home/gpadmin/s3.conf, column open
DROP EXTERNAL TABLE s3regress;
DROP PROTOCOL s3;
gpAux/extensions/gps3ext/regress/s3_regress_schedule
浏览文件 @ 712705e1
......@@ -10,6 +10,8 @@ test: 1_09_partition
test: 1_10_all_regions
test: 1_11_gzipped_data
test: 1_12_no_prefix
test: 1_13_parallel1 1_13_parallel2
test: 1_14_thousands_of_files
test: 2_01_invalid_syntax
test: 2_02_invalid_region
test: 2_03_invalid_config
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或