Skip to content

G
Gpdb

Greenplum / Gpdb

通知 7
Star 1
Fork 0
G
Gpdb

体验新版 GitCode，发现更多精彩内容 >>

提交 ae2a481a 编写于 作者: X Xiaoran Wang
浏览文件
操作

Run gpfdist_ssl test cases on windows 

* Add remote_regress folder in src/bin/gpfdist/
to test gpfdist which runs on a remote host, such
as a windows server. make installcheck_win can
run them.

* The scripts of start_gpfdist_remote_win
are used to start and stop gpfdist runing on windows.
If you want to test gpfdist on another os, such
as aix, you can create a forlder named
start_gpfdist_remote_aix and add some scripts
to start and stop gpfdist.
上级 10052f48
隐藏空白更改
内联 并排
Showing with 14649 addition and 10 deletion
concourse/scripts/ic_gpdb_remote_windows.bash
浏览文件 @ ae2a481a
......@@ -44,10 +44,10 @@ function import_remote_key() {
}
function run_remote_test() {
source ./gpdb_src/gpAux/gpdemo/gpdemo-env.sh
#restart gpdb with ssl
export -f configure_gpdb_ssl
/bin/sh -c "su gpadmin -c configure_gpdb_ssl"
set -eo pipefail
if [ -f /opt/gcc_env.sh ]; then
source /opt/gcc_env.sh
fi
scp -P "${REMOTE_PORT}" ./gpdb_src/concourse/scripts/windows_remote_test.ps1 "${REMOTE_USER}@${REMOTE_HOST}:"
scp -P "${REMOTE_PORT}" ./bin_gpdb_clients_windows/*.msi "${REMOTE_USER}@${REMOTE_HOST}:"
......@@ -61,6 +61,17 @@ function run_remote_test() {
scp -P "${REMOTE_PORT}" ./gpdb_src/concourse/scripts/ic_gpdb_remote_windows.bat "${REMOTE_USER}@${REMOTE_HOST}:"
ssh -T -R"${PGPORT}:127.0.0.1:${PGPORT}" -L8081:127.0.0.1:8081 -L8082:127.0.0.1:8082 -p "${REMOTE_PORT}" "${REMOTE_USER}@${REMOTE_HOST}" "ic_gpdb_remote_windows.bat ${PGPORT}"
# run gpfdist test
pushd gpdb_src/src/test/regress
make pg_regress
popd
pushd gpdb_src/src/bin/gpfdist/remote_regress
echo "export REMOTE_HOST=${REMOTE_HOST}" > ssh_remote_env.sh
echo "export REMOTE_PORT=${REMOTE_PORT}" >> ssh_remote_env.sh
echo "export REMOTE_USER=${REMOTE_USER}" >> ssh_remote_env.sh
echo "export REMOTE_KEY=~/remote.key" >> ssh_remote_env.sh
make installcheck_win
popd
}
function create_cluster() {
......@@ -72,14 +83,30 @@ function create_cluster() {
time make_cluster
}
function _main() {
if [[ -z "${REMOTE_PORT}" ]]; then
function gpadmin_run_tests(){
pushd "${1}"
REMOTE_PORT=${2}
REMOTE_USER=${3}
if [ -z "$REMOTE_PORT" ]; then
REMOTE_PORT=22
fi
yum install -y jq
REMOTE_HOST=$(jq -r '."gpdb-clients-ip"' terraform_windows/metadata)
export REMOTE_HOST
export REMOTE_PORT
export REMOTE_USER
source ./gpdb_src/gpAux/gpdemo/gpdemo-env.sh
source /usr/local/greenplum-db-devel/greenplum_path.sh
configure_gpdb_ssl
time import_remote_key
time run_remote_test
}
function _main() {
export -f configure_gpdb_ssl
export -f import_remote_key
export -f run_remote_test
export -f gpadmin_run_tests
yum install -y jq
pushd bin_gpdb
mv *.tar.gz bin_gpdb.tar.gz
popd
......@@ -89,9 +116,7 @@ function _main() {
popd
time create_cluster
time import_remote_key
time run_remote_test
su gpadmin -c 'gpadmin_run_tests $(pwd) "${REMOTE_PORT}" "${REMOTE_USER}"'
cp bin_gpdb_clients_windows/*.msi bin_gpdb_clients_windows_rc/
pushd bin_gpdb_clients_windows_rc
VERSION=$(cat ../bin_gpdb_clients_windows/version)
......
src/bin/gpfdist/remote_regress/.gitignore 0 → 100644
浏览文件 @ ae2a481a
data/wet_multi_locations_1.tbl
data/wet_multi_locations_2.tbl
data/wet_region.out
sql
expected
results
regression.diffs
regression.out
sh_remote_env.sh
start_gpfdist_remote
data
src/bin/gpfdist/remote_regress/Makefile 0 → 100644
浏览文件 @ ae2a481a
top_builddir = $(shell cd ../../../.. && pwd)
include $(top_builddir)/src/Makefile.global
default: installcheck
REGRESS =
ifeq ($(enable_gpfdist),yes)
ifeq ($(with_openssl),yes)
REGRESS += gpfdist_ssl
endif
endif
PSQLDIR = $(prefix)/bin
REGRESS_OPTS = --init-file=init_file
pre_installcheck:
rm -rf data
mkdir data
cp -rf ../regress/data/* data/
ifeq ($(enable_gpfdist),yes)
ifeq ($(with_openssl),yes)
cp -rf $(MASTER_DATA_DIRECTORY)/gpfdists data/gpfdist_ssl/certs_matching
cp data/gpfdist_ssl/certs_matching/root.crt data/gpfdist_ssl/certs_not_matching
scp -r -P ${REMOTE_PORT} ./data/gpfdist_ssl ${REMOTE_USER}@${REMOTE_HOST}:
endif
endif
installcheck_win: pre_installcheck
rm -rf start_gpfdist_remote
mkdir start_gpfdist_remote
cp -rf start_gpfdist_remote_win/* start_gpfdist_remote
scp -P ${REMOTE_PORT} start_gpfdist_remote/*.bat ${REMOTE_USER}@${REMOTE_HOST}:
$(top_builddir)/src/test/regress/pg_regress --psqldir=$(PSQLDIR) --dbname=gpfdist_regression $(REGRESS) $(REGRESS_OPTS)
clean:
rm -rf regression.* sql results expected
distclean: clean
.PHONY: installcheck clean distclean
src/bin/gpfdist/remote_regress/README 0 → 100644
浏览文件 @ ae2a481a
if gpfdist runs on a remote server, we can run gpdb on a linux
server, then run the regress test cases in this folder on the linux
server to test if the gpfdist can work well.
Now we just support gpfdist on a remote windows server.
make installcheck_win can test gpfdist of windows.
Need to export REMOTE_HOST, REMOTE_PORT, REMOTE_USER, REMOTE_KEY
MASTER_DATA_DIRECTORY, PG_PORT before running these test cases.
The scripts of start_gpfdist_remote_win are used to start and stop gpfdist runing on windows.
If you want to test gpfdist on another os,such as aix, you can create a forlder named
start_gpfdist_remote_aix and add some scripts to start and stop gpfdist.
src/bin/gpfdist/remote_regress/init_file 0 → 100644
浏览文件 @ ae2a481a
-- start_matchignore
m/^DETAIL:.*/
m/^(?:HINT|NOTICE):\s+.+\'DISTRIBUTED BY\' clause.*/
-- end_matchignore
src/bin/gpfdist/remote_regress/input/custom_format.source 0 → 100644
浏览文件 @ ae2a481a
--
-- custom formats for external tables - short and simple functional tests.
--
-- --------------------------------------
-- check platform and start 'gpfdist'
-- --------------------------------------
set optimizer_print_missing_stats = off;
CREATE EXTERNAL WEB TABLE gpfdist_status (x text)
execute E'( python @bindir@/lib/gppinggpfdist.py @hostname@:7575 2>&1 || echo) '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE gpfdist_start (x text)
execute E'((@bindir@/gpfdist -p 7575 -d @abs_srcdir@/data/fixedwidth/ </dev/null >/dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7575 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE gpfdist_stop (x text)
execute E'(ps -A -o pid,comm |grep [g]pfdist |grep -v postgres: |awk \'{print $1;}\' |xargs kill) > /dev/null 2>&1; echo "stopping..."'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
-- start_ignore
-- CREATE EXTERNAL WEB TABLE gpfdist_status (x text)
-- execute E'( python @bindir@/gppinggpfdist.py @hostname@:7575 2>&1 || echo) '
-- on SEGMENT 0
-- FORMAT 'text' (delimiter '|');
SELECT * FROM gpfdist_stop;
-- SELECT * FROM gpfdist_status;
SELECT * FROM gpfdist_start;
-- SELECT * FROM gpfdist_status;
-- end_ignore
-- --------------------------------------
-- the tests
-- --------------------------------------
---------------------------------------------------------------
-- TESTS SUMMARY
-- 1. import data into table with string and numeric fields - data OK
-- 2. import data into table with string and numeric fields - field type error -- no reject limit
-- 3. import data into table with string and numeric fields - too many fields error -- no reject limit
-- 4. import data into table with string and numeric fields - too few fields error -- no reject limit
-- 5. import data into table with string and numeric fields - field too long error -- no reject limit
-- 6. import data into table with string and numeric fields - field type error -- reject limit crossed
-- 7. import data into table with string and numeric fields - field type error -- reject limit not crossed
-- 8. import data into table with string and numeric fields - field type error -- reject limit crossed error written to error table
-- 9. import data into table with string and numeric fields - FORMAT string syntax error -- field definition error
-- 10. import data into table with string and numeric fields - FORMAT string syntax error -- fields order does not corespond to file order
-- canno't be caught when testing string syntax, will appear as a type error and only if the swap was between fields of different type
-- 11. import data into table with string and numeric fields - FORMAT string syntax error -- fields names do not corespond fields in table
-- 12. import data into table with string and numeric fields -- NULL value loaded when NULL value defined
-- 13. import data into table with string and numeric fields -- NULL value loaded when the field has only blanks and preserve_blanks is on
-- 14. import data into table with string and numeric fields - FORMAT string syntax error -- preserve_blanks without NULL definition generates error
-- 15. import data into table with string and numeric fields - FORMAT string syntax error -- preserve_blanks with NULL defined
-- trailing blanks are loaded, blank fields are loaded as strings, only fields with the null value are loaded as NULL
-- 16. import data into table with string and numeric fields - data OK - big input file
-- 17. import data into table with string and numeric fields - data OK - no line_delimiter in input data file
-- 18. import data into table with string and numeric fields - data OK - line delimiter containing several chars
-- 19. MPP-14467 import data from file - field names with spaces
-- 1. import data into table with string and numeric fields - data OK
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_correct.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20',n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 2. import data into table with string and numeric fields - field type error -- no reject limit
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_error_fieldtype.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20',n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 3. import data into table with string and numeric fields - too many fields error -- no reject limit
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_error_toomanyfields.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20',n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 4. import data into table with string and numeric fields - too few fields error -- no reject limit
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_error_toofewfields.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20',n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 5. import data into table with string and numeric fields - field too long error -- no reject limit
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_error_fieldtoolong.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20',n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 6. import data into table with string and numeric fields - field type error -- reject limit crossed
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_error_rejectlimitcrossed.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20',n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15')
SEGMENT REJECT LIMIT 3 ROWS;
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 7. import data into table with string and numeric fields - field type error -- reject limit not crossed
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_error_rejectlimitnotcrossed.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20',n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15')
SEGMENT REJECT LIMIT 5 ROWS;
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 8. import data into table with string and numeric fields - field type error -- reject limit crossed error written to error table
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_error_rejectlimitnotcrossed.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20',n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15')
LOG ERRORS SEGMENT REJECT LIMIT 5 ROWS;
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
SELECT relname,linenum,errmsg FROM gp_read_error_log('tbl_ext_fixedwidth') ORDER BY linenum;
DROP EXTERNAL TABLE tbl_ext_fixedwidth;
-- 9. import data into table with string and numeric fields - FORMAT string syntax error -- field definition error
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_correct.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', '10'=s2, s3='10', dt='20',n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 10. import data into table with string and numeric fields - FORMAT string syntax error -- fields order does not corespond to file order
-- canno't be caught when testing string syntax, will appear as a type error and only if the swap was between fields of different type
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_correct.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', n1='5', s2='10', s3='10', dt='20', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 11. import data into table with string and numeric fields - FORMAT string syntax error -- fields names do not corespond fields in table
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s25 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_correct.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', n1='5', s2='10', s3='10', dt='20', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 12. import data into table with string and numeric fields -- NULL value loaded when NULL value defined
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_nulldefined.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20', n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15', null='NULL');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
SELECT s1 FROM tbl_ext_fixedwidth WHERE s2 IS NULL ORDER BY s1;
SELECT n1 FROM tbl_ext_fixedwidth WHERE n2 IS NULL ORDER BY s1;
-- 13. import data into table with string and numeric fields -- NULL value loaded when the field has only blanks and preserve_blanks is on
-- by default preserve_blanks='on'
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_blankfields.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20', n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
SELECT s1 FROM tbl_ext_fixedwidth WHERE s2 IS NULL ORDER BY s1;
SELECT n1 FROM tbl_ext_fixedwidth WHERE n2 IS NULL ORDER BY s1;
-- 14. import data into table with string and numeric fields - FORMAT string error -- preserve_blanks without NULL definition generates error
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_blankfields.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20', n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15', preserve_blanks='on');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 15. import data into table with string and numeric fields - FORMAT string syntax error -- preserve_blanks with NULL defined
-- trailing blanks are loaded, blank fields are loaded as strings, only fields with the null value are loaded as NULL
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_blankfields_and_nulls.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20', n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15', preserve_blanks='on', null='NULL');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
SELECT s1 FROM tbl_ext_fixedwidth WHERE s2 IS NULL ORDER BY s1;
-- 16. import data into table with string and numeric fields - data OK - big input file
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_big.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20', n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 17. import data into table with string and numeric fields - data OK - no line_delimiter in input data file
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_no_line_delimiter.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20', n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15', line_delim='');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 18. import data into table with string and numeric fields - data OK - line delimiter containing several chars
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_3chars_line_delimiter.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20', n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15', line_delim='@#$');
SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
-- 19. MPP-14467 import data from file - field names with spaces
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
("s 1" char(10), "s 2" varchar(10), s3 text, "dd ttt t t" timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_correct.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', "s 1"='10', "s 2"='10', s3='10', "dd ttt t t"='20',n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM tbl_ext_fixedwidth ORDER BY "s 1";
-- --------------------------------------
-- WRITABLE tests
-- --------------------------------------
-- Helper to remove the output file
DROP EXTERNAL WEB TABLE IF EXISTS clean_out_file;
CREATE EXTERNAL WEB TABLE clean_out_file (x text)
EXECUTE E'( rm -f @abs_srcdir@/data/fixedwidth/fixedwidth_out.tbl)'
on SEGMENT 0
FORMAT 'text';
-- export data into file - all fields have data
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_small_correct.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', s1='10', s2='10', s3='10', dt='20',n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
SELECT * FROM clean_out_file;
CREATE WRITABLE EXTERNAL TABLE tbl_out_fixedwidth
(s1 char(10), s2 varchar(10), s3 text, dt timestamp, n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_out.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_out', s1='10', s2='10', s3='10', dt='25', n1='5', n2='10', n3='10', n4='10', n5='10', n6='10', n7='15');
INSERT INTO tbl_out_fixedwidth SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
\! cat @abs_srcdir@/data/fixedwidth/fixedwidth_out.tbl | wc
-- Execute query to clean out the output file
SELECT * FROM clean_out_file;
-- Create WET with line_delim=E'\n'
DROP EXTERNAL TABLE IF EXISTS tbl_out_fixedwidth;
CREATE WRITABLE EXTERNAL TABLE tbl_out_fixedwidth (
s1 char(10), s2 varchar(10), s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_out.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_out', s1='10',
s2='10', s3='10', dt='25', n1='5', n2='10',
n3='10', n4='10', n5='10', n6='10', n7='15',line_delim=E'\n');
-- Output data
INSERT INTO tbl_out_fixedwidth SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
\! cat @abs_srcdir@/data/fixedwidth/fixedwidth_out.tbl | wc
-- Execute query to clean out the output file
SELECT * FROM clean_out_file;
-- Create WET with line_delim=E'\r\n'
DROP EXTERNAL TABLE IF EXISTS tbl_out_fixedwidth;
CREATE WRITABLE EXTERNAL TABLE tbl_out_fixedwidth (
s1 char(10), s2 varchar(10), s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_out.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_out', s1='10',
s2='10', s3='10', dt='25', n1='5', n2='10',
n3='10', n4='10', n5='10', n6='10', n7='15',line_delim=E'\r\n');
-- Output data
INSERT INTO tbl_out_fixedwidth SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
\! cat @abs_srcdir@/data/fixedwidth/fixedwidth_out.tbl | wc
-- Execute query to clean out the output file
SELECT * FROM clean_out_file;
-- MPP-14492
-- Create WET with line_delim='EndOfLine', which can be arbitrary string
DROP EXTERNAL TABLE IF EXISTS tbl_out_fixedwidth;
CREATE WRITABLE EXTERNAL TABLE tbl_out_fixedwidth (
s1 char(10), s2 varchar(10), s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_out.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_out', s1='10',
s2='10', s3='10', dt='25', n1='5', n2='10',
n3='10', n4='10', n5='10', n6='10', n7='15',line_delim=E'EndOfLine');
-- Output data
INSERT INTO tbl_out_fixedwidth SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
\! cat @abs_srcdir@/data/fixedwidth/fixedwidth_out.tbl | wc
-- Clean out the output file
SELECT * FROM clean_out_file;
-- Create WET with line_delim='qwertyuiopasdfghjklzxcvbnm'
DROP EXTERNAL TABLE IF EXISTS tbl_out_fixedwidth;
CREATE WRITABLE EXTERNAL TABLE tbl_out_fixedwidth (
s1 char(10), s2 varchar(10), s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_out.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_out', s1='10',
s2='10', s3='10', dt='25', n1='5', n2='10',
n3='10', n4='10', n5='10', n6='10', n7='15',line_delim='qwertyuiopasdfghjklzxcvbnm');
-- Output data
INSERT INTO tbl_out_fixedwidth SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
\! cat @abs_srcdir@/data/fixedwidth/fixedwidth_out.tbl | wc
-- Clean out the output file
SELECT * FROM clean_out_file;
-- Create a RET
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
CREATE READABLE EXTERNAL TABLE tbl_ext_fixedwidth (
s1 char(10), s2 varchar(10), s3 text,
col_empty character(5),col_null character varying(5)
)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_null.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_in', null='', preserve_blanks='on',
s1='10',s2='10', s3='10', col_empty='5', col_null='5');
-- Create WET with null string defined: null='NULL'
DROP EXTERNAL TABLE IF EXISTS tbl_out_fixedwidth;
CREATE WRITABLE EXTERNAL TABLE tbl_out_fixedwidth (
s1 char(10), s2 varchar(10), s3 text,
col_empty character(5),col_null character varying(5)
)
LOCATION ('gpfdist://@hostname@:7575/fixedwidth_out.tbl')
FORMAT 'CUSTOM' (formatter='fixedwidth_out', s1='10',s2='10',
s3='10', col_empty='5', col_null='5', null='NULL');
-- export data to output data file
INSERT INTO tbl_out_fixedwidth SELECT * FROM tbl_ext_fixedwidth ORDER BY s1;
\! cat @abs_srcdir@/data/fixedwidth/fixedwidth_out.tbl | grep NULL | wc
-- Clean out the output file
SELECT * FROM clean_out_file;
-- --------------------------------------
-- stop 'gpfdist'
-- --------------------------------------
-- start_ignore
-- SELECT * FROM gpfdist_status;
SELECT * FROM gpfdist_stop;
-- SELECT * FROM gpfdist_status;
-- end_ignore
-- --------------------------------------
-- clean the database
-- --------------------------------------
DROP EXTERNAL TABLE IF EXISTS gpfdist_start;
DROP EXTERNAL TABLE IF EXISTS gpfdist_stop;
DROP EXTERNAL WEB TABLE IF EXISTS clean_out_file;
DROP EXTERNAL TABLE IF EXISTS tbl_ext_fixedwidth;
DROP EXTERNAL TABLE IF EXISTS tbl_out_fixedwidth;
src/bin/gpfdist/remote_regress/input/exttab1.source 0 → 100644
浏览文件 @ ae2a481a
--
-- external tables 1 - short and simple functional tests. The full set of tests
-- exists in cdbunit.
--
set optimizer_print_missing_stats = off;
CREATE TABLE REG_REGION (R_REGIONKEY INT, R_NAME CHAR(25), R_COMMENT VARCHAR(152)) DISTRIBUTED BY (R_REGIONKEY);
-- start_ignore
-- --------------------------------------
-- check platform
-- --------------------------------------
-- drop external web table if exists check_ps;
-- CREATE EXTERNAL WEB TABLE check_ps (x text)
-- execute E'( (ps -ef || ps -aux) | grep gpfdist | grep -v grep)'
-- on SEGMENT 0
-- format 'text';
-- drop external web table if exists check_env;
-- CREATE EXTERNAL WEB TABLE check_env (x text)
-- execute E'( env | sort)'
-- on SEGMENT 0
-- format 'text';
-- select * from check_ps;
-- select * from check_env;
-- end_ignore
-- --------------------------------------
-- 'gpfdist' protocol
-- --------------------------------------
CREATE EXTERNAL WEB TABLE exttab1_gpfdist_status (x text)
execute E'( python @bindir@/lib/gppinggpfdist.py @hostname@:7070 2>&1 || echo) '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE exttab1_gpfdist_start (x text)
execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data </dev/null >/dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE exttab1_gpfdist_stop (x text)
execute E'(ps -A -o pid,comm |grep [g]pfdist |grep -v postgres: |awk \'{print $1;}\' |xargs kill) > /dev/null 2>&1; echo "stopping..."'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
-- start_ignore
select * from exttab1_gpfdist_stop;
select * from exttab1_gpfdist_start;
-- end_ignore
CREATE EXTERNAL TABLE EXT_NATION ( N_NATIONKEY INTEGER ,
N_NAME CHAR(25) ,
N_REGIONKEY INTEGER ,
N_COMMENT VARCHAR(152))
location ('gpfdist://@hostname@:7070/exttab1/nation.tbl' )
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL TABLE EXT_NATION_GZ ( N_NATIONKEY INTEGER ,
N_NAME CHAR(25) ,
N_REGIONKEY INTEGER ,
N_COMMENT VARCHAR(152))
location ('gpfdist://@hostname@:7070/exttab1/nation.tbl.gz' )
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL TABLE EXT_NATION_MATCH ( N_NATIONKEY INTEGER ,
N_NAME CHAR(25) ,
N_REGIONKEY INTEGER ,
N_COMMENT VARCHAR(152))
location ('gpfdist://@hostname@:7070/exttab1/nation.tbl*' )
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL TABLE EXT_REGION (LIKE REG_REGION)
location ('gpfdist://@hostname@:7070/exttab1/region.tbl' )
FORMAT 'text' (delimiter '|');
SELECT * FROM EXT_NATION;
SELECT * FROM EXT_NATION_GZ;
SELECT count(*) FROM EXT_NATION_MATCH;
SELECT * FROM EXT_REGION;
SELECT * FROM EXT_REGION as r, EXT_NATION as n WHERE n.N_REGIONKEY = r.R_REGIONKEY;
-- test: http request header needed for gpfdist
\! wget http://@hostname@:7070/exttab1/missing_fields1.data >/dev/null 2>&1 || echo "Execute error";
\! wget --header='X-GP-PROTO:0' http://@hostname@:7070/exttab1/missing_fields1.data >/dev/null 2>&1 && echo "Execute successully";
\! wget --header='x-gp-proto:1' http://@hostname@:7070/exttab1/missing_fields1.data >/dev/null 2>&1 && echo "Execute successully";
-- test: multiple reader for pipe is forbidden
DROP EXTERNAL WEB TABLE IF EXISTS create_pipe;
CREATE EXTERNAL WEB TABLE create_pipe (x text)
execute E'(rm @abs_srcdir@/data/test_pipe; mkfifo @abs_srcdir@/data/test_pipe) > /dev/null 2>&1; echo "creating pipe..."'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
DROP EXTERNAL WEB TABLE IF EXISTS write_pipe;
CREATE EXTERNAL WEB TABLE write_pipe (x text)
execute E'((for i in `seq 1 5000`;do cat @abs_srcdir@/data/exttab1/nation.tbl;done > @abs_srcdir@/data/test_pipe); echo "writing pipe...")'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
DROP EXTERNAL WEB TABLE IF EXISTS cat_sqlout_result;
CREATE EXTERNAL WEB TABLE cat_sqlout_result (x text)
execute E'(grep -q "Multiple reader to a pipe is forbidden" @abs_srcdir@/data/sql1.out @abs_srcdir@/data/sql2.out && echo "Report error: Multiple reader to a pipe is forbidden";)'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
DROP EXTERNAL TABLE IF EXISTS pipe_ext1;
CREATE EXTERNAL TABLE pipe_ext1(LIKE EXT_NATION)
location ('gpfdist://@hostname@:7070/test_pipe' )
FORMAT 'text' (delimiter '|');
DROP EXTERNAL TABLE IF EXISTS pipe_ext2;
CREATE EXTERNAL TABLE pipe_ext2(LIKE EXT_NATION)
location ('file://@hostname@@abs_srcdir@/data/test_pipe' )
FORMAT 'text' (delimiter '|');
select * from create_pipe;
\! psql -d gpfdist_regression -c 'select count(*) from pipe_ext1;' > @abs_srcdir@/data/sql1.out 2>&1 &
\! psql -d gpfdist_regression -c 'select count(*) from pipe_ext2;' > @abs_srcdir@/data/sql2.out 2>&1 &
select * from write_pipe;
select * from cat_sqlout_result;
-- start_ignore
select * from exttab1_gpfdist_stop;
-- end_ignore
-- drop tables
DROP EXTERNAL TABLE EXT_NATION;
DROP EXTERNAL TABLE EXT_NATION_GZ;
DROP EXTERNAL TABLE EXT_NATION_MATCH;
DROP EXTERNAL TABLE EXT_REGION;
--
-- gpfdist in csv (mpp-1519, etc)
--
CREATE EXTERNAL WEB TABLE gpfdist_csv_start (x text)
execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data </dev/null >/dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
--
-- "
-- (end the double quote so emacs highlighting works correctly)
--
-- start_ignore
select * from exttab1_gpfdist_stop;
select * from gpfdist_csv_start;
-- end_ignore
create external table ext_whois (
source_lineno int,
domain_name varchar(350),
subdomain varchar(150),
tld varchar(50),
ip_address inet,
ip_address_int bigint,
reverse_dns varchar(512),
reverse_domain varchar(350),
registrar varchar(200),
registrar_referral varchar(512),
whois_server varchar(512),
harvest_date timestamp with time zone,
created_on timestamp with time zone,
update_date timestamp with time zone,
expire_date timestamp with time zone,
rank int,
status char(1),
update_status int,
nameserver1 varchar(512),
nameserver1_domain_name varchar(350),
nameserver1_subdomain varchar(150),
nameserver1_tld varchar(50),
nameserver2 varchar(512),
nameserver2_domain_name varchar(350),
nameserver2_subdomain varchar(150),
nameserver2_tld varchar(50),
nameserver3 varchar(512),
nameserver3_domain_name varchar(350),
nameserver3_subdomain varchar(150),
nameserver3_tld varchar(50),
nameserver4 varchar(512),
nameserver4_domain_name varchar(350),
nameserver4_subdomain varchar(150),
nameserver4_tld varchar(50),
nameserver5 varchar(512),
nameserver5_domain_name varchar(350),
nameserver5_subdomain varchar(150),
nameserver5_tld varchar(50),
registrant_name varchar(200),
registrant_organization varchar(200),
registrant_email varchar(512),
registrant_email_domain varchar(350),
registrant_email_subdomain varchar(150),
registrant_email_tld varchar(50),
registrant_phone varchar(50),
registrant_fax varchar(50),
registrant_addrall varchar(1024),
registrant_street1 varchar(200),
registrant_street2 varchar(200),
registrant_street3 varchar(200),
registrant_city varchar(200),
registrant_state_province varchar(100),
registrant_postal_code varchar(50),
registrant_country varchar(100),
tech_name varchar(200),
tech_organization varchar(200),
tech_email varchar(512),
tech_email_domain varchar(350),
tech_email_subdomain varchar(150),
tech_email_tld varchar(50),
tech_phone varchar(50),
tech_fax varchar(50),
tech_addrall varchar(1024),
tech_street1 varchar(200),
tech_street2 varchar(200),
tech_street3 varchar(200),
tech_city varchar(200),
tech_state_province varchar(100),
tech_postal_code varchar(50),
tech_country varchar(100),
admin_name varchar(200),
admin_organization varchar(200),
admin_email varchar(512),
admin_email_domain varchar(350),
admin_email_subdomain varchar(150),
admin_email_tld varchar(50),
admin_phone varchar(50),
admin_fax varchar(50),
admin_addrall varchar(1024),
admin_street1 varchar(200),
admin_street2 varchar(200),
admin_street3 varchar(200),
admin_city varchar(200),
admin_state_province varchar(100),
admin_postal_code varchar(50),
admin_country varchar(100),
rec_path varchar(512),
raw_record text
)
location ('gpfdist://@hostname@:7070/exttab1/whois.csv' )
format 'csv' ( quote as '"' header);
-- "
select count(*) from ext_whois;
create table v_w as select * from ext_whois;
-- sort distinct spill to disk/external sort
select count(*) from (select * from v_w union select * from ext_whois) as foo;
select count(*) from (select * from v_w union all select * from ext_whois) as foo;
select count(*) from (select distinct * from ext_whois) as foo;
select count(*) from (select * from ext_whois order by 1 limit 300000 offset 30000) as foo;
select count(*) from (select * from ext_whois order by 1 limit 300000 offset 10) as foo;
select source_lineno, domain_name from ext_whois order by
source_lineno,
domain_name,
subdomain,
tld,
ip_address,
ip_address_int,
reverse_dns,
reverse_domain,
registrar,
registrar_referral,
whois_server,
harvest_date,
created_on,
update_date,
expire_date,
rank,
status,
update_status,
nameserver1,
nameserver1_domain_name,
nameserver1_subdomain,
nameserver1_tld,
nameserver2,
nameserver2_domain_name,
nameserver2_subdomain,
nameserver2_tld,
nameserver3,
nameserver3_domain_name,
nameserver3_subdomain,
nameserver3_tld,
nameserver4,
nameserver4_domain_name ,
nameserver4_subdomain,
nameserver4_tld,
nameserver5,
nameserver5_domain_name,
nameserver5_subdomain,
nameserver5_tld,
registrant_name,
registrant_organization ,
registrant_email,
registrant_email_domain,
registrant_email_subdomain,
registrant_email_tld,
registrant_phone,
registrant_fax,
registrant_addrall,
registrant_street1,
registrant_street2,
registrant_street3,
registrant_city,
registrant_state_province,
registrant_postal_code,
registrant_country,
tech_name,
tech_organization,
tech_email,
tech_email_domain,
tech_email_subdomain,
tech_email_tld,
tech_phone,
tech_fax,
tech_addrall,
tech_street1,
tech_street2,
tech_street3,
tech_city,
tech_state_province,
tech_postal_code,
tech_country,
admin_name,
admin_organization,
admin_email,
admin_email_domain ,
admin_email_subdomain,
admin_email_tld,
admin_phone,
admin_fax,
admin_addrall,
admin_street1,
admin_street2,
admin_street3,
admin_city,
admin_state_province,
admin_postal_code,
admin_country,
rec_path,
raw_record limit 10 offset 10;
-- test FILL EMPTY FIELDS
create external table ext_fill1(a int, b int, c int)
location('gpfdist://@hostname@:7070/exttab1/missing_fields1.data' )
format 'text' ( delimiter '|' fill missing fields);
create external table ext_fill2(a int, b int, c int)
location('gpfdist://@hostname@:7070/exttab1/missing_fields2.data' )
format 'text' ( delimiter '|' fill missing fields);
-- also test force not null in addition to fill empty fields
create external table ext_fill3_fnn(a text, b text, c text)
location('gpfdist://@hostname@:7070/exttab1/missing_fields1.data' )
format 'csv' ( delimiter '|' fill missing fields force not null c);
create external table ext_fill4_fnn(a text, b text, c text)
location('gpfdist://@hostname@:7070/exttab1/missing_fields1.data' )
format 'csv' ( delimiter '|' fill missing fields force not null b,c);
select * from ext_fill1 order by a,b,c; -- should pad missing attrs with nulls
select * from ext_fill2 order by a,b,c; -- should fail due to empty data line
select * from ext_fill3_fnn where c is null; -- should be empty
select * from ext_fill3_fnn where b is null; -- one row result
select * from ext_fill4_fnn where c is null or b is null; -- should be empty
drop external table ext_fill1;
drop external table ext_fill2;
drop external table ext_fill3_fnn;
drop external table ext_fill4_fnn;
-- start_ignore
drop external table if exists ext_newline1;
drop external table if exists ext_newline2;
-- end_ignore
-- test NEWLINE
create external table ext_newline1(N_NATIONKEY INT, N_NAME text, N_REGIONKEY INT, N_COMMENT text)
location('gpfdist://@hostname@:7070/exttab1/nation.tbl' )
format 'text' ( delimiter '|' newline 'lf');
create external table ext_newline2(like ext_newline1)
location('gpfdist://@hostname@:7070/exttab1/nation.tbl' )
format 'text' ( delimiter '|' newline 'cr');
create external table ext_willfail(a int)
location('gpfdist://@hostname@:7070/exttab1/nation.tbl' )
format 'text' ( delimiter '|' newline 'blah'); -- should fail with invalid newline.
select * from ext_newline1; -- should pass. using the correct linefeed. file has 'lf'.
select * from ext_newline2; -- should fail. using an incorrect linefeed. file has 'lf'.
drop external table ext_newline1;
drop external table ext_newline2;
-- MPP-6698
create external table ext_mpp6698(a text)
location('gpfdist://@hostname@:7070/exttab1/mpp6698.data' )
format 'csv' (quote '''');
select * from ext_mpp6698 order by a; -- should ignore the quotes
drop external table ext_mpp6698;
-- MPP-12839
create external table ext_mpp12839
(
userguid varchar(36),
action_time timestamp without time zone,
action_type smallint,
object_id integer,
general_param character varying(8000),
plu_id smallint,
page_name character varying(2000),
browser integer,
platform integer,
filtering_data character varying(8000),
is_new_user boolean,
dummy character varying(1)
)
location('gpfdist://@hostname@:7070/exttab1/mpp12839*.data' )
format 'text' (delimiter E'\177' null '' newline 'crlf')
segment reject limit 100 rows;
select * from ext_mpp12839; -- should not reach reject limit
drop external table ext_mpp12839;
--
-- bad csv (quote must be a single char)
--
create external table bad_whois (
source_lineno int,
domain_name varchar(350)
)
location ('gpfdist://@hostname@:7070/exttab1/whois.csv' )
format 'csv' ( header quote as 'ggg');
select count(*) from bad_whois;
drop external table bad_whois;
--
-- Test invalid locations
--
create external table bad_location1 (a int4, b text)
location ('gpfdist://@hostname@:7070/exttab1/non_existent_dir/*' )
format 'csv';
select count(*) from bad_location1;
drop external table bad_location1;
create external table bad_location2 (a int4, b text)
location ('gpfdist://@hostname@:7070/exttab1/non_existent_file.txt' )
format 'csv';
select count(*) from bad_location2;
drop external table bad_location2;
--
-- WET tests
--
--
-- CREATE (including LIKE, DISTRIBUTED BY)
--
-- positive
create writable external table wet_pos1(a text, b text) location('gpfdist://@hostname@:7070/wet.out') format 'text';
create writable external table wet_pos2(a text, b text) location('gpfdist://@hostname@:7070/wet.out') format 'text' distributed by(b);
create writable external table wet_pos3(like wet_pos2) location('gpfdist://@hostname@:7070/wet.out') format 'text' distributed by(a,b);
create writable external table wet_region(like reg_region) location('gpfdist://@hostname@:7070/wet_region.out') format 'text';
create readable external table ret_region(like wet_region) location('gpfdist://@hostname@:7070/wet_region.out') format 'text';
--
-- WET: export some data with INSERT SELECT, INSERT and COPY.
--
COPY reg_region FROM STDIN DELIMITER '|';
0|AFRICA|lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to
1|AMERICA|hs use ironic, even requests. s
\.
INSERT INTO wet_region SELECT * from reg_region;
COPY wet_region FROM STDIN DELIMITER '|';
2|ASIA|ges. thinly even pinto beans ca
3|EUROPE|ly final courts cajole furiously final excuse
\.
INSERT INTO wet_region VALUES(4,'MIDDLE EAST','uickly special');
--
-- Now use RET to see if data was exported correctly.
-- NOTE: since we don't bother cleaning up the exported file, it may grow bigger
-- in between runs, so we don't check for count(*), and instead check for distinct.
--
SELECT DISTINCT(R_NAME) FROM ret_region ORDER BY R_NAME;
-- start_ignore
DROP TABLE IF EXISTS table_multi_locations;
DROP EXTERNAL TABLE IF EXISTS ext_table_multi_locations;
-- end_ignore
-- The use of more than one gpfdist. Each gpfdist points to a different file to write into
CREATE TABLE table_multi_locations (a int, b text) DISTRIBUTED BY (a);
INSERT INTO table_multi_locations VALUES (generate_series(1,256), 'test_1');
CREATE WRITABLE EXTERNAL TABLE ext_table_multi_locations (a int, b text) LOCATION ('gpfdist://@hostname@:7070/wet_multi_locations_1.tbl', 'gpfdist://@hostname@:7070/wet_multi_locations_2.tbl') FORMAT 'TEXT' (DELIMITER AS '|' NULL AS 'null');
INSERT INTO ext_table_multi_locations SELECT * FROM table_multi_locations;
DROP TABLE IF EXISTS table_multi_locations;
DROP EXTERNAL TABLE IF EXISTS ext_table_multi_locations;
-- start_ignore
select * from exttab1_gpfdist_stop;
-- end_ignore
--
-- get an error for missing gpfdist
--
select count(*) from ext_whois;
drop external table ext_whois;
drop external table exttab1_gpfdist_start;
drop external table gpfdist_csv_start;
drop external table exttab1_gpfdist_stop;
drop external table wet_pos1;
drop external table wet_pos2;
drop external table wet_pos3;
drop external table wet_region;
drop external table ret_region;
drop table reg_region;
src/bin/gpfdist/remote_regress/input/gpfdist2.source 0 → 100644
浏览文件 @ ae2a481a
--
-- GPFDIST test cases set 2. This test set is moved from cdbunit.
--
set optimizer_print_missing_stats = off;
CREATE TABLE REG_REGION (R_REGIONKEY INT, R_NAME CHAR(25), R_COMMENT VARCHAR(152)) DISTRIBUTED BY (R_REGIONKEY);
-- start_ignore
-- --------------------------------------
-- check platform
-- --------------------------------------
-- drop external web table if exists check_ps;
-- CREATE EXTERNAL WEB TABLE check_ps (x text)
-- execute E'( (ps -ef || ps -aux) | grep gpfdist | grep -v grep)'
-- on SEGMENT 0
-- format 'text';
-- drop external web table if exists check_env;
-- CREATE EXTERNAL WEB TABLE check_env (x text)
-- execute E'( env | sort)'
-- on SEGMENT 0
-- format 'text';
-- select * from check_ps;
-- select * from check_env;
-- end_ignore
-- --------------------------------------
-- 'gpfdist' protocol
-- --------------------------------------
CREATE EXTERNAL WEB TABLE gpfdist2_start (x text)
execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data </dev/null >/dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE gpfdist2_stop (x text)
execute E'(ps -A -o pid,comm |grep [g]pfdist |grep -v postgres: |awk \'{print $1;}\' |xargs kill) > /dev/null 2>&1; echo "stopping..."'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
-- start_ignore
select * from gpfdist2_stop;
select * from gpfdist2_start;
-- end_ignore
-- start_ignore
drop table if exists lineitem;
drop external table if exists ext_lineitem;
-- end_ignore
-- test 1 using a .gz file
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.gz'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 2 using a .bz2 file
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.bz2'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 4 column data type does not match
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(2),
L_SHIPMODE CHAR(1),
L_COMMENT VARCHAR(4)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 5 data has extra or missing attributes
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(2),
L_LINESTATUS CHAR(2),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(2),
L_LINESTATUS CHAR(2),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44),
L_BLAH CHAR(1)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 6 external table attributes (encoding, escape, etc)
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
NULL AS ''
ESCAPE AS '#'
)
ENCODING 'UTF8'
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 7 external table attributes (escape off)
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
ESCAPE 'OFF'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 8 external table attributes csv (force not null)
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'csv'
(
DELIMITER AS '|'
FORCE NOT NULL L_COMMENT
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 9
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.csv.header'
)
FORMAT 'csv'
(
DELIMITER AS '|'
QUOTE AS '"'
HEADER
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 10
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/bad_data/lineitem.tbl.bad'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
LOG ERRORS SEGMENT REJECT LIMIT 2
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 11
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/invalid_filename.txt'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 12
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl',
'gpfdist://@hostname@:7070/gpfdist2/more_data/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 13
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.*'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 14
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/bad_data/lineitem.tbl.bad'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
LOG ERRORS SEGMENT REJECT LIMIT 200
;
SELECT count(*) FROM ext_lineitem;
\t
DROP EXTERNAL TABLE ext_lineitem;
-- test 15
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/data/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER '|'
)
;
SELECT DISTINCT L_ORDERKEY FROM ext_lineitem WHERE L_ORDERKEY > 10 GROUP BY L_ORDERKEY ORDER BY L_ORDERKEY LIMIT 10 OFFSET 2;
DROP EXTERNAL TABLE ext_lineitem;
-- test 16
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER '|'
)
;
CREATE INDEX index ON ext_lineitem (L_ORDERKEY);
TRUNCATE ext_lineitem;
DELETE FROM ext_lineitem where L_ORDERKEY > 10;
UPDATE ext_lineitem SET L_ORDERKEY = 10 where L_ORDERKEY > 10;
INSERT INTO ext_lineitem VALUES (1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1);
DROP EXTERNAL TABLE ext_lineitem;
-- test 17 different delimiter
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/more_data/lineitem.tbl.space'
)
FORMAT 'text'
(
DELIMITER ' '
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/more_data/lineitem.tbl.comma'
)
FORMAT 'text'
(
DELIMITER ','
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/more_data/lineitem.tbl.carrot'
)
FORMAT 'text'
(
DELIMITER '^'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 18 without header row and header is defined in format clause
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'csv'
(
DELIMITER AS '|'
QUOTE AS '"'
HEADER
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 19 with header row but header is not defined in format clause
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.csv.header'
)
FORMAT 'csv'
(
DELIMITER AS '|'
QUOTE AS '"'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 20 without header row and header is not defined in format clause
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'csv'
(
DELIMITER AS '|'
QUOTE AS '"'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 22 table creation commands
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
CREATE TABLE lineitem AS SELECT * FROM ext_lineitem;
SELECT count(*) FROM lineitem;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
DROP TABLE lineitem;
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT * INTO lineitem FROM ext_lineitem;
SELECT count(*) FROM lineitem;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
DROP TABLE lineitem;
-- test 23 external table attributes text (force not null)
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
FORCE NOT NULL L_COMMENT
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
-- test 24 inherit external table
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.gz'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
CREATE EXTERNAL TABLE ext_lineitem_copy (LIKE ext_lineitem)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.gz'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem_copy;
DROP EXTERNAL TABLE ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem_copy;
-- test 25 line too long with defaults
CREATE EXTERNAL TABLE ext_test (
id text,
stuff text
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/longline.txt'
)
FORMAT 'text'
(
DELIMITER AS ','
)
;
SELECT count(*) FROM ext_test;
DROP EXTERNAL TABLE ext_test;
CREATE EXTERNAL TABLE ext_test1 (
id text,
stuff text
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/longline.csv'
)
FORMAT 'csv'
(
DELIMITER AS ','
)
;
SELECT count(*) FROM ext_test1;
DROP EXTERNAL TABLE ext_test1;
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem_cr.tbl'
)
FORMAT 'csv'
(
DELIMITER AS '|'
NEWLINE 'CR'
)
;
SELECT count(*) FROM ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem;
CREATE EXTERNAL TABLE ext_crlf_with_lf_column(c1 int, c2 text) LOCATION ('gpfdist://@hostname@:7070/gpfdist2/crlf_with_lf_column.csv') FORMAT 'csv' (NEWLINE 'CRLF');
SELECT count(*) FROM ext_crlf_with_lf_column;
DROP EXTERNAL TABLE ext_crlf_with_lf_column;
CREATE EXTERNAL TABLE ext_crlf_with_lf_column(c1 int, c2 text) LOCATION ('gpfdist://@hostname@:7070/gpfdist2/crlf_with_lf_column.csv') FORMAT 'text' (DELIMITER ',' NEWLINE 'CRLF');
SELECT count(*) FROM ext_crlf_with_lf_column;
DROP EXTERNAL TABLE ext_crlf_with_lf_column;
-- start_ignore
select * from gpfdist2_stop;
-- end_ignore
src/bin/gpfdist/remote_regress/input/gpfdist_ssl.source 0 → 100644
浏览文件 @ ae2a481a
--
-- GPFDISTS test cases
--
-- start_ignore
-- --------------------------------------
-- check platform
-- --------------------------------------
-- drop external web table if exists check_ps;
-- CREATE EXTERNAL WEB TABLE check_ps (x text)
-- execute E'( (ps -ef || ps -aux) | grep gpfdist | grep -v grep)'
-- on SEGMENT 0
-- format 'text';
-- drop external web table if exists check_env;
-- CREATE EXTERNAL WEB TABLE check_env (x text)
-- execute E'( env | sort)'
-- on SEGMENT 0
-- format 'text';
-- select * from check_ps;
-- select * from check_env;
-- end_ignore
-- --------------------------------------
-- 'gpfdists' protocol
-- --------------------------------------
CREATE EXTERNAL WEB TABLE gpfdist_ssl_start (x text)
execute E'((source @abs_srcdir@/ssh_remote_env.sh; @abs_srcdir@/start_gpfdist_remote/start_gpfdist_with_ssl.bash); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE gpfdist_ssl_not_matching_start (x text)
execute E'((source @abs_srcdir@/ssh_remote_env.sh; @abs_srcdir@/start_gpfdist_remote/start_gpfdist_ssl_not_matching.bash); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE gpfdist_ssl_stop (x text)
execute E'((source @abs_srcdir@/ssh_remote_env.sh;@abs_srcdir@/start_gpfdist_remote/stop_gpfdist.bash); echo "stopping...")'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
-- start_ignore
select * from gpfdist_ssl_stop;
select * from gpfdist_ssl_start;
-- end_ignore
-- Helper to remove the output file
DROP EXTERNAL WEB TABLE IF EXISTS clean_out_file;
CREATE EXTERNAL WEB TABLE clean_out_file (x text)
EXECUTE E'(rm -f @abs_srcdir@/data/gpfdist_ssl/tbl2.tbl)'
on SEGMENT 0
FORMAT 'text';
-- Execute query to clean out the output file
SELECT * FROM clean_out_file;
-- gpfdist_ssl case 1
DROP TABLE IF EXISTS tbl_on_heap;
CREATE TABLE tbl_on_heap (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision);
INSERT INTO tbl_on_heap VALUES
('aaa','twoa','shpits','2011-06-01 12:30:30',23,732,834567,45.67,789.123,7.12345,123.456789),
('bbb','twob','shpits','2011-06-01 12:30:30',23,732,834567,45.67,789.123,7.12345,123.456789),
('ccc','twoc','shpits','2011-06-01 12:30:30',23,732,834567,45.67,789.123,7.12345,123.456789 );
DROP EXTERNAL TABLE IF EXISTS tbl;
CREATE WRITABLE EXTERNAL TABLE tbl (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl2.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
INSERT INTO tbl SELECT * FROM tbl_on_heap;
SELECT * FROM tbl_on_heap ORDER BY s1;
-- gpfdist_ssl case 2
DROP TABLE IF EXISTS tbl_on_heap2;
CREATE TABLE tbl_on_heap2 (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision);
DROP EXTERNAL TABLE IF EXISTS tbl2;
CREATE EXTERNAL TABLE tbl2 (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl2.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
INSERT INTO tbl_on_heap2 SELECT * FROM tbl2;
SELECT * FROM tbl_on_heap2 ORDER BY s1;
-- gpfdist_ssl case 3
-- on same ETL server
DROP TABLE IF EXISTS tbl_on_heap;
CREATE TABLE tbl_on_heap (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision);
DROP EXTERNAL TABLE IF EXISTS tbl;
CREATE EXTERNAL TABLE tbl (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl','gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl2.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
INSERT INTO tbl_on_heap SELECT * FROM tbl;
SELECT * FROM tbl_on_heap ORDER BY s1;
-- gpfdist_ssl case 4
DROP TABLE IF EXISTS tbl_on_heap;
CREATE TABLE tbl_on_heap (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision);
DROP EXTERNAL TABLE IF EXISTS tbl;
CREATE EXTERNAL TABLE tbl (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
INSERT INTO tbl_on_heap SELECT * FROM tbl;
SELECT * FROM tbl_on_heap ORDER BY s1;
-- gpfdist_ssl case 5
DROP TABLE IF EXISTS tbl_on_heap;
CREATE TABLE tbl_on_heap (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision);
DROP EXTERNAL TABLE IF EXISTS tbl;
CREATE EXTERNAL TABLE tbl (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl','gpfdist://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
DROP TABLE IF EXISTS tbl_on_heap;
-- start_ignore
select * from gpfdist_ssl_stop;
select * from gpfdist_ssl_not_matching_start;
-- end_ignore
-- gpfdist_ssl case 6
DROP EXTERNAL TABLE IF EXISTS tbl;
CREATE EXTERNAL TABLE tbl (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
SELECT * FROM tbl;
SET verify_gpfdists_cert=off;
SELECT * FROM tbl;
-- start_ignore
select * from gpfdist_ssl_stop;
-- end_ignore
src/bin/gpfdist/remote_regress/output/custom_format.source 0 → 100644
浏览文件 @ ae2a481a
因为 它太大了无法显示 source diff 。你可以改为 查看blob。
src/bin/gpfdist/remote_regress/output/exttab1.source 0 → 100644
浏览文件 @ ae2a481a
--
-- external tables 1 - short and simple functional tests. The full set of tests
-- exists in cdbunit.
--
set optimizer_print_missing_stats = off;
CREATE TABLE REG_REGION (R_REGIONKEY INT, R_NAME CHAR(25), R_COMMENT VARCHAR(152)) DISTRIBUTED BY (R_REGIONKEY);
-- start_ignore
-- end_ignore
-- --------------------------------------
-- 'gpfdist' protocol
-- --------------------------------------
CREATE EXTERNAL WEB TABLE exttab1_gpfdist_status (x text)
execute E'( python @bindir@/lib/gppinggpfdist.py @hostname@:7070 2>&1 || echo) '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE exttab1_gpfdist_start (x text)
execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data </dev/null >/dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE exttab1_gpfdist_stop (x text)
execute E'(ps -A -o pid,comm |grep [g]pfdist |grep -v postgres: |awk \'{print $1;}\' |xargs kill) > /dev/null 2>&1; echo "stopping..."'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
-- start_ignore
-- end_ignore
CREATE EXTERNAL TABLE EXT_NATION ( N_NATIONKEY INTEGER ,
N_NAME CHAR(25) ,
N_REGIONKEY INTEGER ,
N_COMMENT VARCHAR(152))
location ('gpfdist://@hostname@:7070/exttab1/nation.tbl' )
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL TABLE EXT_NATION_GZ ( N_NATIONKEY INTEGER ,
N_NAME CHAR(25) ,
N_REGIONKEY INTEGER ,
N_COMMENT VARCHAR(152))
location ('gpfdist://@hostname@:7070/exttab1/nation.tbl.gz' )
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL TABLE EXT_NATION_MATCH ( N_NATIONKEY INTEGER ,
N_NAME CHAR(25) ,
N_REGIONKEY INTEGER ,
N_COMMENT VARCHAR(152))
location ('gpfdist://@hostname@:7070/exttab1/nation.tbl*' )
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL TABLE EXT_REGION (LIKE REG_REGION)
location ('gpfdist://@hostname@:7070/exttab1/region.tbl' )
FORMAT 'text' (delimiter '|');
SELECT * FROM EXT_NATION;
n_nationkey | n_name | n_regionkey | n_comment
-------------+---------------------------+-------------+--------------------------------------------------------------------------------------------------------------------
0 | ALGERIA | 0 | haggle. carefully final deposits detect slyly agai
1 | ARGENTINA | 1 | al foxes promise slyly according to the regular accounts. bold requests alon
2 | BRAZIL | 1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special
3 | CANADA | 1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold
4 | EGYPT | 4 | y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d
5 | ETHIOPIA | 0 | ven packages wake quickly. regu
6 | FRANCE | 3 | refully final requests. regular, ironi
7 | GERMANY | 3 | l platelets. regular accounts x-ray: unusual, regular acco
8 | INDIA | 2 | ss excuses cajole slyly across the packages. deposits print aroun
9 | INDONESIA | 2 | slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull
10 | IRAN | 4 | efully alongside of the slyly final dependencies.
11 | IRAQ | 4 | nic deposits boost atop the quickly final requests? quickly regula
12 | JAPAN | 2 | ously. final, express gifts cajole a
13 | JORDAN | 4 | ic deposits are blithely about the carefully regular pa
14 | KENYA | 0 | pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t
15 | MOROCCO | 0 | rns. blithely bold courts among the closely regular packages use furiously bold platelets?
16 | MOZAMBIQUE | 0 | s. ironic, unusual asymptotes wake blithely r
17 | PERU | 1 | platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun
18 | CHINA | 2 | c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos
19 | ROMANIA | 3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account
20 | SAUDI ARABIA | 4 | ts. silent requests haggle. closely express packages sleep across the blithely
21 | VIETNAM | 2 | hely enticingly express accounts. even, final
22 | RUSSIA | 3 | requests against the platelets use never according to the quickly regular pint
23 | UNITED KINGDOM | 3 | eans boost carefully special requests. accounts are. carefull
24 | UNITED STATES | 1 | y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be
(25 rows)
SELECT * FROM EXT_NATION_GZ;
n_nationkey | n_name | n_regionkey | n_comment
-------------+---------------------------+-------------+--------------------------------------------------------------------------------------------------------------------
0 | ALGERIA | 0 | haggle. carefully final deposits detect slyly agai
1 | ARGENTINA | 1 | al foxes promise slyly according to the regular accounts. bold requests alon
2 | BRAZIL | 1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special
3 | CANADA | 1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold
4 | EGYPT | 4 | y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d
5 | ETHIOPIA | 0 | ven packages wake quickly. regu
6 | FRANCE | 3 | refully final requests. regular, ironi
7 | GERMANY | 3 | l platelets. regular accounts x-ray: unusual, regular acco
8 | INDIA | 2 | ss excuses cajole slyly across the packages. deposits print aroun
9 | INDONESIA | 2 | slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull
10 | IRAN | 4 | efully alongside of the slyly final dependencies.
11 | IRAQ | 4 | nic deposits boost atop the quickly final requests? quickly regula
12 | JAPAN | 2 | ously. final, express gifts cajole a
13 | JORDAN | 4 | ic deposits are blithely about the carefully regular pa
14 | KENYA | 0 | pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t
15 | MOROCCO | 0 | rns. blithely bold courts among the closely regular packages use furiously bold platelets?
16 | MOZAMBIQUE | 0 | s. ironic, unusual asymptotes wake blithely r
17 | PERU | 1 | platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun
18 | CHINA | 2 | c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos
19 | ROMANIA | 3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account
20 | SAUDI ARABIA | 4 | ts. silent requests haggle. closely express packages sleep across the blithely
21 | VIETNAM | 2 | hely enticingly express accounts. even, final
22 | RUSSIA | 3 | requests against the platelets use never according to the quickly regular pint
23 | UNITED KINGDOM | 3 | eans boost carefully special requests. accounts are. carefull
24 | UNITED STATES | 1 | y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be
(25 rows)
SELECT count(*) FROM EXT_NATION_MATCH;
count
-------
50
(1 row)
SELECT * FROM EXT_REGION;
r_regionkey | r_name | r_comment
-------------+---------------------------+---------------------------------------------------------------------------------------------------------------------
0 | AFRICA | lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to
1 | AMERICA | hs use ironic, even requests. s
2 | ASIA | ges. thinly even pinto beans ca
3 | EUROPE | ly final courts cajole furiously final excuse
4 | MIDDLE EAST | uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl
(5 rows)
SELECT * FROM EXT_REGION as r, EXT_NATION as n WHERE n.N_REGIONKEY = r.R_REGIONKEY;
r_regionkey | r_name | r_comment | n_nationkey | n_name | n_regionkey | n_comment
-------------+---------------------------+---------------------------------------------------------------------------------------------------------------------+-------------+---------------------------+-------------+--------------------------------------------------------------------------------------------------------------------
0 | AFRICA | lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to | 16 | MOZAMBIQUE | 0 | s. ironic, unusual asymptotes wake blithely r
0 | AFRICA | lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to | 15 | MOROCCO | 0 | rns. blithely bold courts among the closely regular packages use furiously bold platelets?
0 | AFRICA | lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to | 14 | KENYA | 0 | pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t
0 | AFRICA | lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to | 5 | ETHIOPIA | 0 | ven packages wake quickly. regu
0 | AFRICA | lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to | 0 | ALGERIA | 0 | haggle. carefully final deposits detect slyly agai
1 | AMERICA | hs use ironic, even requests. s | 24 | UNITED STATES | 1 | y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be
1 | AMERICA | hs use ironic, even requests. s | 17 | PERU | 1 | platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun
1 | AMERICA | hs use ironic, even requests. s | 3 | CANADA | 1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold
1 | AMERICA | hs use ironic, even requests. s | 2 | BRAZIL | 1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special
1 | AMERICA | hs use ironic, even requests. s | 1 | ARGENTINA | 1 | al foxes promise slyly according to the regular accounts. bold requests alon
3 | EUROPE | ly final courts cajole furiously final excuse | 23 | UNITED KINGDOM | 3 | eans boost carefully special requests. accounts are. carefull
3 | EUROPE | ly final courts cajole furiously final excuse | 22 | RUSSIA | 3 | requests against the platelets use never according to the quickly regular pint
3 | EUROPE | ly final courts cajole furiously final excuse | 19 | ROMANIA | 3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account
3 | EUROPE | ly final courts cajole furiously final excuse | 7 | GERMANY | 3 | l platelets. regular accounts x-ray: unusual, regular acco
3 | EUROPE | ly final courts cajole furiously final excuse | 6 | FRANCE | 3 | refully final requests. regular, ironi
2 | ASIA | ges. thinly even pinto beans ca | 21 | VIETNAM | 2 | hely enticingly express accounts. even, final
2 | ASIA | ges. thinly even pinto beans ca | 18 | CHINA | 2 | c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos
2 | ASIA | ges. thinly even pinto beans ca | 12 | JAPAN | 2 | ously. final, express gifts cajole a
2 | ASIA | ges. thinly even pinto beans ca | 9 | INDONESIA | 2 | slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull
2 | ASIA | ges. thinly even pinto beans ca | 8 | INDIA | 2 | ss excuses cajole slyly across the packages. deposits print aroun
4 | MIDDLE EAST | uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl | 20 | SAUDI ARABIA | 4 | ts. silent requests haggle. closely express packages sleep across the blithely
4 | MIDDLE EAST | uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl | 13 | JORDAN | 4 | ic deposits are blithely about the carefully regular pa
4 | MIDDLE EAST | uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl | 11 | IRAQ | 4 | nic deposits boost atop the quickly final requests? quickly regula
4 | MIDDLE EAST | uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl | 10 | IRAN | 4 | efully alongside of the slyly final dependencies.
4 | MIDDLE EAST | uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl | 4 | EGYPT | 4 | y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d
(25 rows)
-- test: http request header needed for gpfdist
\! wget http://@hostname@:7070/exttab1/missing_fields1.data >/dev/null 2>&1 || echo "Execute error";
Execute error
\! wget --header='X-GP-PROTO:0' http://@hostname@:7070/exttab1/missing_fields1.data >/dev/null 2>&1 && echo "Execute successully";
Execute successully
\! wget --header='x-gp-proto:1' http://@hostname@:7070/exttab1/missing_fields1.data >/dev/null 2>&1 && echo "Execute successully";
Execute successully
-- test: multiple reader for pipe is forbidden
DROP EXTERNAL WEB TABLE IF EXISTS create_pipe;
-- start_ignore
NOTICE: table "create_pipe" does not exist, skipping
-- end_ignore
CREATE EXTERNAL WEB TABLE create_pipe (x text)
execute E'(rm @abs_srcdir@/data/test_pipe; mkfifo @abs_srcdir@/data/test_pipe) > /dev/null 2>&1; echo "creating pipe..."'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
DROP EXTERNAL WEB TABLE IF EXISTS write_pipe;
-- start_ignore
NOTICE: table "write_pipe" does not exist, skipping
-- end_ignore
CREATE EXTERNAL WEB TABLE write_pipe (x text)
execute E'((for i in `seq 1 5000`;do cat @abs_srcdir@/data/exttab1/nation.tbl;done > @abs_srcdir@/data/test_pipe); echo "writing pipe...")'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
DROP EXTERNAL WEB TABLE IF EXISTS cat_sqlout_result;
-- start_ignore
NOTICE: table "cat_sqlout_result" does not exist, skipping
-- end_ignore
CREATE EXTERNAL WEB TABLE cat_sqlout_result (x text)
execute E'(grep -q "Multiple reader to a pipe is forbidden" @abs_srcdir@/data/sql1.out @abs_srcdir@/data/sql2.out && echo "Report error: Multiple reader to a pipe is forbidden";)'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
DROP EXTERNAL TABLE IF EXISTS pipe_ext1;
-- start_ignore
NOTICE: table "pipe_ext1" does not exist, skipping
-- end_ignore
CREATE EXTERNAL TABLE pipe_ext1(LIKE EXT_NATION)
location ('gpfdist://@hostname@:7070/test_pipe' )
FORMAT 'text' (delimiter '|');
DROP EXTERNAL TABLE IF EXISTS pipe_ext2;
-- start_ignore
NOTICE: table "pipe_ext2" does not exist, skipping
-- end_ignore
CREATE EXTERNAL TABLE pipe_ext2(LIKE EXT_NATION)
location ('file://@hostname@@abs_srcdir@/data/test_pipe' )
FORMAT 'text' (delimiter '|');
select * from create_pipe;
x
------------------
creating pipe...
(1 row)
\! psql -d gpfdist_regression -c 'select count(*) from pipe_ext1;' > @abs_srcdir@/data/sql1.out 2>&1 &
\! psql -d gpfdist_regression -c 'select count(*) from pipe_ext2;' > @abs_srcdir@/data/sql2.out 2>&1 &
select * from write_pipe;
x
-----------------
writing pipe...
(1 row)
select * from cat_sqlout_result;
x
------------------------------------------------------
Report error: Multiple reader to a pipe is forbidden
(1 row)
-- start_ignore
-- end_ignore
-- drop tables
DROP EXTERNAL TABLE EXT_NATION;
DROP EXTERNAL TABLE EXT_NATION_GZ;
DROP EXTERNAL TABLE EXT_NATION_MATCH;
DROP EXTERNAL TABLE EXT_REGION;
--
-- gpfdist in csv (mpp-1519, etc)
--
CREATE EXTERNAL WEB TABLE gpfdist_csv_start (x text)
execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data </dev/null >/dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
--
-- "
-- (end the double quote so emacs highlighting works correctly)
--
-- start_ignore
-- end_ignore
create external table ext_whois (
source_lineno int,
domain_name varchar(350),
subdomain varchar(150),
tld varchar(50),
ip_address inet,
ip_address_int bigint,
reverse_dns varchar(512),
reverse_domain varchar(350),
registrar varchar(200),
registrar_referral varchar(512),
whois_server varchar(512),
harvest_date timestamp with time zone,
created_on timestamp with time zone,
update_date timestamp with time zone,
expire_date timestamp with time zone,
rank int,
status char(1),
update_status int,
nameserver1 varchar(512),
nameserver1_domain_name varchar(350),
nameserver1_subdomain varchar(150),
nameserver1_tld varchar(50),
nameserver2 varchar(512),
nameserver2_domain_name varchar(350),
nameserver2_subdomain varchar(150),
nameserver2_tld varchar(50),
nameserver3 varchar(512),
nameserver3_domain_name varchar(350),
nameserver3_subdomain varchar(150),
nameserver3_tld varchar(50),
nameserver4 varchar(512),
nameserver4_domain_name varchar(350),
nameserver4_subdomain varchar(150),
nameserver4_tld varchar(50),
nameserver5 varchar(512),
nameserver5_domain_name varchar(350),
nameserver5_subdomain varchar(150),
nameserver5_tld varchar(50),
registrant_name varchar(200),
registrant_organization varchar(200),
registrant_email varchar(512),
registrant_email_domain varchar(350),
registrant_email_subdomain varchar(150),
registrant_email_tld varchar(50),
registrant_phone varchar(50),
registrant_fax varchar(50),
registrant_addrall varchar(1024),
registrant_street1 varchar(200),
registrant_street2 varchar(200),
registrant_street3 varchar(200),
registrant_city varchar(200),
registrant_state_province varchar(100),
registrant_postal_code varchar(50),
registrant_country varchar(100),
tech_name varchar(200),
tech_organization varchar(200),
tech_email varchar(512),
tech_email_domain varchar(350),
tech_email_subdomain varchar(150),
tech_email_tld varchar(50),
tech_phone varchar(50),
tech_fax varchar(50),
tech_addrall varchar(1024),
tech_street1 varchar(200),
tech_street2 varchar(200),
tech_street3 varchar(200),
tech_city varchar(200),
tech_state_province varchar(100),
tech_postal_code varchar(50),
tech_country varchar(100),
admin_name varchar(200),
admin_organization varchar(200),
admin_email varchar(512),
admin_email_domain varchar(350),
admin_email_subdomain varchar(150),
admin_email_tld varchar(50),
admin_phone varchar(50),
admin_fax varchar(50),
admin_addrall varchar(1024),
admin_street1 varchar(200),
admin_street2 varchar(200),
admin_street3 varchar(200),
admin_city varchar(200),
admin_state_province varchar(100),
admin_postal_code varchar(50),
admin_country varchar(100),
rec_path varchar(512),
raw_record text
)
location ('gpfdist://@hostname@:7070/exttab1/whois.csv' )
format 'csv' ( quote as '"' header);
NOTICE: HEADER means that each one of the data files has a header row
-- "
select count(*) from ext_whois;
count
-------
23
(1 row)
create table v_w as select * from ext_whois;
-- sort distinct spill to disk/external sort
select count(*) from (select * from v_w union select * from ext_whois) as foo;
count
-------
23
(1 row)
select count(*) from (select * from v_w union all select * from ext_whois) as foo;
count
-------
46
(1 row)
select count(*) from (select distinct * from ext_whois) as foo;
count
-------
23
(1 row)
select count(*) from (select * from ext_whois order by 1 limit 300000 offset 30000) as foo;
count
-------
0
(1 row)
select count(*) from (select * from ext_whois order by 1 limit 300000 offset 10) as foo;
count
-------
13
(1 row)
select source_lineno, domain_name from ext_whois order by
source_lineno,
domain_name,
subdomain,
tld,
ip_address,
ip_address_int,
reverse_dns,
reverse_domain,
registrar,
registrar_referral,
whois_server,
harvest_date,
created_on,
update_date,
expire_date,
rank,
status,
update_status,
nameserver1,
nameserver1_domain_name,
nameserver1_subdomain,
nameserver1_tld,
nameserver2,
nameserver2_domain_name,
nameserver2_subdomain,
nameserver2_tld,
nameserver3,
nameserver3_domain_name,
nameserver3_subdomain,
nameserver3_tld,
nameserver4,
nameserver4_domain_name ,
nameserver4_subdomain,
nameserver4_tld,
nameserver5,
nameserver5_domain_name,
nameserver5_subdomain,
nameserver5_tld,
registrant_name,
registrant_organization ,
registrant_email,
registrant_email_domain,
registrant_email_subdomain,
registrant_email_tld,
registrant_phone,
registrant_fax,
registrant_addrall,
registrant_street1,
registrant_street2,
registrant_street3,
registrant_city,
registrant_state_province,
registrant_postal_code,
registrant_country,
tech_name,
tech_organization,
tech_email,
tech_email_domain,
tech_email_subdomain,
tech_email_tld,
tech_phone,
tech_fax,
tech_addrall,
tech_street1,
tech_street2,
tech_street3,
tech_city,
tech_state_province,
tech_postal_code,
tech_country,
admin_name,
admin_organization,
admin_email,
admin_email_domain ,
admin_email_subdomain,
admin_email_tld,
admin_phone,
admin_fax,
admin_addrall,
admin_street1,
admin_street2,
admin_street3,
admin_city,
admin_state_province,
admin_postal_code,
admin_country,
rec_path,
raw_record limit 10 offset 10;
source_lineno | domain_name
---------------+-----------------
596 | 007dy.net
650 | 00163.net
725 | 0-sexyteens.net
782 | 0000ff.net
839 | 0041.net
903 | 0086-595.net
967 | 000gps.net
1019 | 0086-534.net
1083 | 0-l.net
1137 | 0049165.net
(10 rows)
-- test FILL EMPTY FIELDS
create external table ext_fill1(a int, b int, c int)
location('gpfdist://@hostname@:7070/exttab1/missing_fields1.data' )
format 'text' ( delimiter '|' fill missing fields);
create external table ext_fill2(a int, b int, c int)
location('gpfdist://@hostname@:7070/exttab1/missing_fields2.data' )
format 'text' ( delimiter '|' fill missing fields);
-- also test force not null in addition to fill empty fields
create external table ext_fill3_fnn(a text, b text, c text)
location('gpfdist://@hostname@:7070/exttab1/missing_fields1.data' )
format 'csv' ( delimiter '|' fill missing fields force not null c);
create external table ext_fill4_fnn(a text, b text, c text)
location('gpfdist://@hostname@:7070/exttab1/missing_fields1.data' )
format 'csv' ( delimiter '|' fill missing fields force not null b,c);
select * from ext_fill1 order by a,b,c; -- should pad missing attrs with nulls
a | b | c
---+---+---
1 | 1 | 1
2 | 2 |
3 | |
4 | 4 |
5 | 5 | 5
(5 rows)
select * from ext_fill2 order by a,b,c; -- should fail due to empty data line
ERROR: missing data for column "b", found empty data line (seg0 slice1 @hostname@:50000 pid=64819)
CONTEXT: External table ext_fill2, line 3 of gpfdist://@hostname@:7070/exttab1/missing_fields2.data: ""
select * from ext_fill3_fnn where c is null; -- should be empty
a | b | c
---+---+---
(0 rows)
select * from ext_fill3_fnn where b is null; -- one row result
a | b | c
---+---+---
3 | |
(1 row)
select * from ext_fill4_fnn where c is null or b is null; -- should be empty
a | b | c
---+---+---
(0 rows)
drop external table ext_fill1;
drop external table ext_fill2;
drop external table ext_fill3_fnn;
drop external table ext_fill4_fnn;
-- test NEWLINE
create external table ext_newline1(N_NATIONKEY INT, N_NAME text, N_REGIONKEY INT, N_COMMENT text)
location('gpfdist://@hostname@:7070/exttab1/nation.tbl' )
format 'text' ( delimiter '|' newline 'lf');
create external table ext_newline2(like ext_newline1)
location('gpfdist://@hostname@:7070/exttab1/nation.tbl' )
format 'text' ( delimiter '|' newline 'cr');
create external table ext_willfail(a int)
location('gpfdist://@hostname@:7070/exttab1/nation.tbl' )
format 'text' ( delimiter '|' newline 'blah'); -- should fail with invalid newline.
ERROR: invalid value for NEWLINE "blah"
HINT: Valid options are: 'LF', 'CRLF' and 'CR'.
select * from ext_newline1; -- should pass. using the correct linefeed. file has 'lf'.
n_nationkey | n_name | n_regionkey | n_comment
-------------+----------------+-------------+--------------------------------------------------------------------------------------------------------------------
0 | ALGERIA | 0 | haggle. carefully final deposits detect slyly agai
1 | ARGENTINA | 1 | al foxes promise slyly according to the regular accounts. bold requests alon
2 | BRAZIL | 1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special
3 | CANADA | 1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold
4 | EGYPT | 4 | y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d
5 | ETHIOPIA | 0 | ven packages wake quickly. regu
6 | FRANCE | 3 | refully final requests. regular, ironi
7 | GERMANY | 3 | l platelets. regular accounts x-ray: unusual, regular acco
8 | INDIA | 2 | ss excuses cajole slyly across the packages. deposits print aroun
9 | INDONESIA | 2 | slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull
10 | IRAN | 4 | efully alongside of the slyly final dependencies.
11 | IRAQ | 4 | nic deposits boost atop the quickly final requests? quickly regula
12 | JAPAN | 2 | ously. final, express gifts cajole a
13 | JORDAN | 4 | ic deposits are blithely about the carefully regular pa
14 | KENYA | 0 | pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t
15 | MOROCCO | 0 | rns. blithely bold courts among the closely regular packages use furiously bold platelets?
16 | MOZAMBIQUE | 0 | s. ironic, unusual asymptotes wake blithely r
17 | PERU | 1 | platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun
18 | CHINA | 2 | c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos
19 | ROMANIA | 3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account
20 | SAUDI ARABIA | 4 | ts. silent requests haggle. closely express packages sleep across the blithely
21 | VIETNAM | 2 | hely enticingly express accounts. even, final
22 | RUSSIA | 3 | requests against the platelets use never according to the quickly regular pint
23 | UNITED KINGDOM | 3 | eans boost carefully special requests. accounts are. carefull
24 | UNITED STATES | 1 | y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be
(25 rows)
select * from ext_newline2; -- should fail. using an incorrect linefeed. file has 'lf'.
ERROR: extra data after last expected column (seg0 slice1 @hostname@:50000 pid=64819)
CONTEXT: External table ext_newline2, line 1 of gpfdist://@hostname@:7070/exttab1/nation.tbl: "0|ALGERIA|0| haggle. carefully final deposits detect slyly agai
1|ARGENTINA|1|al foxes promise slyly..."
drop external table ext_newline1;
drop external table ext_newline2;
-- MPP-6698
create external table ext_mpp6698(a text)
location('gpfdist://@hostname@:7070/exttab1/mpp6698.data' )
format 'csv' (quote '''');
select * from ext_mpp6698 order by a; -- should ignore the quotes
a
----------------------------------------------------
first row
second row with embedded single quote ' here
third row with another embedded quote at the end '
(3 rows)
drop external table ext_mpp6698;
-- MPP-12839
create external table ext_mpp12839
(
userguid varchar(36),
action_time timestamp without time zone,
action_type smallint,
object_id integer,
general_param character varying(8000),
plu_id smallint,
page_name character varying(2000),
browser integer,
platform integer,
filtering_data character varying(8000),
is_new_user boolean,
dummy character varying(1)
)
location('gpfdist://@hostname@:7070/exttab1/mpp12839*.data' )
format 'text' (delimiter E'\177' null '' newline 'crlf')
segment reject limit 100 rows;
select * from ext_mpp12839; -- should not reach reject limit
NOTICE: found 6 data formatting errors (6 or more input rows), rejected related input data
userguid | action_time | action_type | object_id | general_param | plu_id | page_name | browser | platform | filtering_data | is_new_user | dummy
----------+-------------+-------------+-----------+---------------+--------+-----------+---------+----------+----------------+-------------+-------
(0 rows)
drop external table ext_mpp12839;
--
-- bad csv (quote must be a single char)
--
create external table bad_whois (
source_lineno int,
domain_name varchar(350)
)
location ('gpfdist://@hostname@:7070/exttab1/whois.csv' )
format 'csv' ( header quote as 'ggg');
ERROR: quote must be a single one-byte character
select count(*) from bad_whois;
ERROR: relation "bad_whois" does not exist
LINE 1: select count(*) from bad_whois;
^
drop external table bad_whois;
ERROR: table "bad_whois" does not exist
--
-- Test invalid locations
--
create external table bad_location1 (a int4, b text)
location ('gpfdist://@hostname@:7070/exttab1/non_existent_dir/*' )
format 'csv';
select count(*) from bad_location1;
ERROR: http response code 404 from gpfdist (gpfdist://@hostname@:7070/exttab1/non_existent_dir/*): HTTP/1.0 404 file not found
drop external table bad_location1;
create external table bad_location2 (a int4, b text)
location ('gpfdist://@hostname@:7070/exttab1/non_existent_file.txt' )
format 'csv';
select count(*) from bad_location2;
ERROR: http response code 404 from gpfdist (gpfdist://@hostname@:7070/exttab1/non_existent_file.txt): HTTP/1.0 404 file not found
drop external table bad_location2;
--
-- WET tests
--
--
-- CREATE (including LIKE, DISTRIBUTED BY)
--
-- positive
create writable external table wet_pos1(a text, b text) location('gpfdist://@hostname@:7070/wet.out') format 'text';
create writable external table wet_pos2(a text, b text) location('gpfdist://@hostname@:7070/wet.out') format 'text' distributed by(b);
create writable external table wet_pos3(like wet_pos2) location('gpfdist://@hostname@:7070/wet.out') format 'text' distributed by(a,b);
create writable external table wet_region(like reg_region) location('gpfdist://@hostname@:7070/wet_region.out') format 'text';
NOTICE: table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table
create readable external table ret_region(like wet_region) location('gpfdist://@hostname@:7070/wet_region.out') format 'text';
--
-- WET: export some data with INSERT SELECT, INSERT and COPY.
--
COPY reg_region FROM STDIN DELIMITER '|';
INSERT INTO wet_region SELECT * from reg_region;
COPY wet_region FROM STDIN DELIMITER '|';
INSERT INTO wet_region VALUES(4,'MIDDLE EAST','uickly special');
--
-- Now use RET to see if data was exported correctly.
-- NOTE: since we don't bother cleaning up the exported file, it may grow bigger
-- in between runs, so we don't check for count(*), and instead check for distinct.
--
SELECT DISTINCT(R_NAME) FROM ret_region ORDER BY R_NAME;
r_name
---------------------------
AFRICA
AMERICA
ASIA
EUROPE
MIDDLE EAST
(5 rows)
-- start_ignore
DROP TABLE IF EXISTS table_multi_locations;
NOTICE: table "table_multi_locations" does not exist, skipping
DROP EXTERNAL TABLE IF EXISTS ext_table_multi_locations;
NOTICE: table "ext_table_multi_locations" does not exist, skipping
-- end_ignore
-- The use of more than one gpfdist. Each gpfdist points to a different file to write into
CREATE TABLE table_multi_locations (a int, b text) DISTRIBUTED BY (a);
INSERT INTO table_multi_locations VALUES (generate_series(1,256), 'test_1');
CREATE WRITABLE EXTERNAL TABLE ext_table_multi_locations (a int, b text) LOCATION ('gpfdist://@hostname@:7070/wet_multi_locations_1.tbl', 'gpfdist://@hostname@:7070/wet_multi_locations_2.tbl') FORMAT 'TEXT' (DELIMITER AS '|' NULL AS 'null');
INSERT INTO ext_table_multi_locations SELECT * FROM table_multi_locations;
DROP TABLE IF EXISTS table_multi_locations;
DROP EXTERNAL TABLE IF EXISTS ext_table_multi_locations;
-- start_ignore
-- end_ignore
--
-- get an error for missing gpfdist
--
select count(*) from ext_whois;
ERROR: connection with gpfdist failed for gpfdist://@hostname@:7070/whois.csv. effective url: http://127.0.0.1:7070/whois.csv. error code = 61 (Connection refused) (seg1 slice1 @hostname@:50001 pid=64820)
drop external table ext_whois;
drop external table exttab1_gpfdist_start;
drop external table gpfdist_csv_start;
drop external table exttab1_gpfdist_stop;
drop external table wet_pos1;
drop external table wet_pos2;
drop external table wet_pos3;
drop external table wet_region;
drop external table ret_region;
drop table reg_region;
src/bin/gpfdist/remote_regress/output/gpfdist2.source 0 → 100644
浏览文件 @ ae2a481a
--
-- GPFDIST test cases set 2. This test set is moved from cdbunit.
--
set optimizer_print_missing_stats = off;
CREATE TABLE REG_REGION (R_REGIONKEY INT, R_NAME CHAR(25), R_COMMENT VARCHAR(152)) DISTRIBUTED BY (R_REGIONKEY);
-- start_ignore
-- end_ignore
-- --------------------------------------
-- 'gpfdist' protocol
-- --------------------------------------
CREATE EXTERNAL WEB TABLE gpfdist2_start (x text)
execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data </dev/null >/dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE gpfdist2_stop (x text)
execute E'(ps -A -o pid,comm |grep [g]pfdist |grep -v postgres: |awk \'{print $1;}\' |xargs kill) > /dev/null 2>&1; echo "stopping..."'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
-- start_ignore
select * from gpfdist2_stop;
x
-------------
stopping...
(1 row)
select * from gpfdist2_start;
x
-------------
starting...
(1 row)
-- end_ignore
-- start_ignore
drop table if exists lineitem;
NOTICE: table "lineitem" does not exist, skipping
drop external table if exists ext_lineitem;
NOTICE: table "ext_lineitem" does not exist, skipping
-- end_ignore
-- test 1 using a .gz file
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.gz'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
count
-------
256
(1 row)
DROP EXTERNAL TABLE ext_lineitem;
-- test 2 using a .bz2 file
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.bz2'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
count
-------
256
(1 row)
DROP EXTERNAL TABLE ext_lineitem;
-- test 4 column data type does not match
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(2),
L_SHIPMODE CHAR(1),
L_COMMENT VARCHAR(4)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
ERROR: value too long for type character(2) (seg0 slice1 172.17.0.4:25432 pid=36415)
DETAIL: External table ext_lineitem, line 1 of gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl, column l_shipinstruct
DROP EXTERNAL TABLE ext_lineitem;
-- test 5 data has extra or missing attributes
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(2),
L_LINESTATUS CHAR(2),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
ERROR: extra data after last expected column (seg0 slice1 172.17.0.4:25432 pid=36415)
DETAIL: External table ext_lineitem, line 1 of gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl: "1|156|4|1|17|17954.55|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular ..."
DROP EXTERNAL TABLE ext_lineitem;
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(2),
L_LINESTATUS CHAR(2),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44),
L_BLAH CHAR(1)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
ERROR: missing data for column "l_blah" (seg0 slice1 172.17.0.4:25432 pid=36415)
DETAIL: External table ext_lineitem, line 1 of gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl: "1|156|4|1|17|17954.55|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular ..."
DROP EXTERNAL TABLE ext_lineitem;
-- test 6 external table attributes (encoding, escape, etc)
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
NULL AS ''
ESCAPE AS '#'
)
ENCODING 'UTF8'
;
SELECT count(*) FROM ext_lineitem;
count
-------
256
(1 row)
DROP EXTERNAL TABLE ext_lineitem;
-- test 7 external table attributes (escape off)
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
ESCAPE 'OFF'
)
;
SELECT count(*) FROM ext_lineitem;
count
-------
256
(1 row)
DROP EXTERNAL TABLE ext_lineitem;
-- test 8 external table attributes csv (force not null)
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'csv'
(
DELIMITER AS '|'
FORCE NOT NULL L_COMMENT
)
;
SELECT count(*) FROM ext_lineitem;
count
-------
256
(1 row)
DROP EXTERNAL TABLE ext_lineitem;
-- test 9
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.csv.header'
)
FORMAT 'csv'
(
DELIMITER AS '|'
QUOTE AS '"'
HEADER
)
;
NOTICE: HEADER means that each one of the data files has a header row
SELECT count(*) FROM ext_lineitem;
count
-------
255
(1 row)
DROP EXTERNAL TABLE ext_lineitem;
-- test 10
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/bad_data/lineitem.tbl.bad'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
LOG ERRORS SEGMENT REJECT LIMIT 2
;
SELECT count(*) FROM ext_lineitem;
ERROR: segment reject limit reached, aborting operation
DETAIL: Last error was: invalid input syntax for integer: "a68", column l_partkey (seg1 slice1 172.17.0.4:25433 pid=36416)
CONTEXT: External table ext_lineitem, line 2 of gpfdist://@hostname@:7070/gpfdist2/bad_data/lineitem.tbl.bad, column l_partkey
DROP EXTERNAL TABLE ext_lineitem;
-- test 11
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/invalid_filename.txt'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
ERROR: http response code 404 from gpfdist (gpfdist://@hostname@:7070/gpfdist2/invalid_filename.txt): HTTP/1.0 404 file not found (seg0 slice1 172.17.0.4:25432 pid=36415)
DROP EXTERNAL TABLE ext_lineitem;
-- test 12
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl',
'gpfdist://@hostname@:7070/gpfdist2/more_data/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
count
-------
264
(1 row)
DROP EXTERNAL TABLE ext_lineitem;
-- test 13
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.*'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
ERROR: invalid input syntax for integer: "HEADER" (seg0 slice1 127.0.0.1:25435 pid=67886)
DETAIL: External table ext_lineitem, line 1 of gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.*, column l_orderkey
DROP EXTERNAL TABLE ext_lineitem;
-- test 14
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/bad_data/lineitem.tbl.bad'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
LOG ERRORS SEGMENT REJECT LIMIT 200
;
SELECT count(*) FROM ext_lineitem;
NOTICE: found 4 data formatting errors (4 or more input rows), rejected related input data
count
-------
28
(1 row)
\t
DROP EXTERNAL TABLE ext_lineitem;
-- test 15
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/data/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER '|'
)
;
SELECT DISTINCT L_ORDERKEY FROM ext_lineitem WHERE L_ORDERKEY > 10 GROUP BY L_ORDERKEY ORDER BY L_ORDERKEY LIMIT 10 OFFSET 2;
ERROR: http response code 404 from gpfdist (gpfdist://@hostname@:7070/gpfdist2/data/lineitem.tbl): HTTP/1.0 404 file not found (seg0 slice1 172.17.0.4:25432 pid=36454)
DROP EXTERNAL TABLE ext_lineitem;
-- test 16
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER '|'
)
;
CREATE INDEX index ON ext_lineitem (L_ORDERKEY);
ERROR: cannot create indexes on external tables
TRUNCATE ext_lineitem;
ERROR: "ext_lineitem" is an external relation and can't be truncated
DELETE FROM ext_lineitem where L_ORDERKEY > 10;
ERROR: cannot update or delete from external relation "ext_lineitem"
UPDATE ext_lineitem SET L_ORDERKEY = 10 where L_ORDERKEY > 10;
ERROR: cannot update or delete from external relation "ext_lineitem"
INSERT INTO ext_lineitem VALUES (1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1);
ERROR: cannot change a readable external table "ext_lineitem"
DROP EXTERNAL TABLE ext_lineitem;
-- test 17 different delimiter
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/more_data/lineitem.tbl.space'
)
FORMAT 'text'
(
DELIMITER ' '
)
;
SELECT count(*) FROM ext_lineitem;
1
DROP EXTERNAL TABLE ext_lineitem;
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/more_data/lineitem.tbl.comma'
)
FORMAT 'text'
(
DELIMITER ','
)
;
SELECT count(*) FROM ext_lineitem;
1
DROP EXTERNAL TABLE ext_lineitem;
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/more_data/lineitem.tbl.carrot'
)
FORMAT 'text'
(
DELIMITER '^'
)
;
SELECT count(*) FROM ext_lineitem;
1
DROP EXTERNAL TABLE ext_lineitem;
-- test 18 without header row and header is defined in format clause
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'csv'
(
DELIMITER AS '|'
QUOTE AS '"'
HEADER
)
;
NOTICE: HEADER means that each one of the data files has a header row
SELECT count(*) FROM ext_lineitem;
255
DROP EXTERNAL TABLE ext_lineitem;
-- test 19 with header row but header is not defined in format clause
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.csv.header'
)
FORMAT 'csv'
(
DELIMITER AS '|'
QUOTE AS '"'
)
;
SELECT count(*) FROM ext_lineitem;
ERROR: invalid input syntax for integer: "HEADER" (seg1 slice1 127.0.0.1:25433 pid=67887)
DETAIL: External table ext_lineitem, line 1 of gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.csv.header, column l_orderkey
DROP EXTERNAL TABLE ext_lineitem;
-- test 20 without header row and header is not defined in format clause
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'csv'
(
DELIMITER AS '|'
QUOTE AS '"'
)
;
SELECT count(*) FROM ext_lineitem;
256
DROP EXTERNAL TABLE ext_lineitem;
-- test 22 table creation commands
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
CREATE TABLE lineitem AS SELECT * FROM ext_lineitem;
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'l_orderkey' as the Greenplum Database data distribution key for this table.
HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
SELECT count(*) FROM lineitem;
256
SELECT count(*) FROM ext_lineitem;
256
DROP EXTERNAL TABLE ext_lineitem;
DROP TABLE lineitem;
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT * INTO lineitem FROM ext_lineitem;
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'l_orderkey' as the Greenplum Database data distribution key for this table.
HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
SELECT count(*) FROM lineitem;
256
SELECT count(*) FROM ext_lineitem;
256
DROP EXTERNAL TABLE ext_lineitem;
DROP TABLE lineitem;
-- test 23 external table attributes text (force not null)
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl'
)
FORMAT 'text'
(
DELIMITER AS '|'
FORCE NOT NULL L_COMMENT
)
;
ERROR: force not null available only in CSV mode
SELECT count(*) FROM ext_lineitem;
ERROR: relation "ext_lineitem" does not exist
LINE 1: SELECT count(*) FROM ext_lineitem;
^
DROP EXTERNAL TABLE ext_lineitem;
ERROR: table "ext_lineitem" does not exist
-- test 24 inherit external table
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.gz'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem;
256
CREATE EXTERNAL TABLE ext_lineitem_copy (LIKE ext_lineitem)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.gz'
)
FORMAT 'text'
(
DELIMITER AS '|'
)
;
SELECT count(*) FROM ext_lineitem_copy;
256
DROP EXTERNAL TABLE ext_lineitem;
DROP EXTERNAL TABLE ext_lineitem_copy;
-- test 25 line too long with defaults
CREATE EXTERNAL TABLE ext_test (
id text,
stuff text
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/longline.txt'
)
FORMAT 'text'
(
DELIMITER AS ','
)
;
SELECT count(*) FROM ext_test;
ERROR: gpfdist error - line too long in file @abs_srcdir@/data/gpfdist2/longline.txt near (0 bytes) (seg1 slice1 172.17.0.4:25433 pid=36416)
DETAIL: External table ext_test, file gpfdist://@hostname@:7070/gpfdist2/longline.txt
DROP EXTERNAL TABLE ext_test;
CREATE EXTERNAL TABLE ext_test1 (
id text,
stuff text
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/longline.csv'
)
FORMAT 'csv'
(
DELIMITER AS ','
)
;
SELECT count(*) FROM ext_test1;
ERROR: gpfdist error - line too long in file @abs_srcdir@/data/gpfdist2/longline.csv near (0 bytes) (seg0 slice1 172.17.0.4:25432 pid=36415)
DETAIL: External table ext_test1, file gpfdist://@hostname@:7070/gpfdist2/longline.csv
DROP EXTERNAL TABLE ext_test1;
CREATE EXTERNAL TABLE ext_lineitem (
L_ORDERKEY INT8,
L_PARTKEY INTEGER,
L_SUPPKEY INTEGER,
L_LINENUMBER integer,
L_QUANTITY decimal,
L_EXTENDEDPRICE decimal,
L_DISCOUNT decimal,
L_TAX decimal,
L_RETURNFLAG CHAR(1),
L_LINESTATUS CHAR(1),
L_SHIPDATE date,
L_COMMITDATE date,
L_RECEIPTDATE date,
L_SHIPINSTRUCT CHAR(25),
L_SHIPMODE CHAR(10),
L_COMMENT VARCHAR(44)
)
LOCATION
(
'gpfdist://@hostname@:7070/gpfdist2/lineitem_cr.tbl'
)
FORMAT 'csv'
(
DELIMITER AS '|'
NEWLINE 'CR'
)
;
SELECT count(*) FROM ext_lineitem;
2985
DROP EXTERNAL TABLE ext_lineitem;
CREATE EXTERNAL TABLE ext_crlf_with_lf_column(c1 int, c2 text) LOCATION ('gpfdist://@hostname@:7070/gpfdist2/crlf_with_lf_column.csv') FORMAT 'csv' (NEWLINE 'CRLF');
SELECT count(*) FROM ext_crlf_with_lf_column;
10367
DROP EXTERNAL TABLE ext_crlf_with_lf_column;
CREATE EXTERNAL TABLE ext_crlf_with_lf_column(c1 int, c2 text) LOCATION ('gpfdist://@hostname@:7070/gpfdist2/crlf_with_lf_column.csv') FORMAT 'text' (DELIMITER ',' NEWLINE 'CRLF');
SELECT count(*) FROM ext_crlf_with_lf_column;
10367
DROP EXTERNAL TABLE ext_crlf_with_lf_column;
-- start_ignore
select * from gpfdist2_stop;
stopping...
-- end_ignore
src/bin/gpfdist/remote_regress/output/gpfdist_ssl.source 0 → 100644
浏览文件 @ ae2a481a
--
-- GPFDISTS test cases
--
-- start_ignore
-- end_ignore
-- --------------------------------------
-- 'gpfdists' protocol
-- --------------------------------------
CREATE EXTERNAL WEB TABLE gpfdist_ssl_start (x text)
execute E'((source @abs_srcdir@/ssh_remote_env.sh; @abs_srcdir@/start_gpfdist_remote/start_gpfdist_with_ssl.bash); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE gpfdist_ssl_not_matching_start (x text)
execute E'((source @abs_srcdir@/ssh_remote_env.sh; @abs_srcdir@/start_gpfdist_remote/start_gpfdist_ssl_not_matching.bash); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") '
on SEGMENT 0
FORMAT 'text' (delimiter '|');
CREATE EXTERNAL WEB TABLE gpfdist_ssl_stop (x text)
execute E'((source @abs_srcdir@/ssh_remote_env.sh;@abs_srcdir@/start_gpfdist_remote/stop_gpfdist.bash); echo "stopping...")'
on SEGMENT 0
FORMAT 'text' (delimiter '|');
-- start_ignore
select * from gpfdist_ssl_stop;
x
-------------
stopping...
(1 row)
select * from gpfdist_ssl_start;
x
-------------
starting...
(1 row)
-- end_ignore
-- Helper to remove the output file
DROP EXTERNAL WEB TABLE IF EXISTS clean_out_file;
NOTICE: table "clean_out_file" does not exist, skipping
CREATE EXTERNAL WEB TABLE clean_out_file (x text)
EXECUTE E'(rm -f @abs_srcdir@/data/gpfdist_ssl/tbl2.tbl)'
on SEGMENT 0
FORMAT 'text';
-- Execute query to clean out the output file
SELECT * FROM clean_out_file;
x
---
(0 rows)
-- gpfdist_ssl case 1
DROP TABLE IF EXISTS tbl_on_heap;
NOTICE: table "tbl_on_heap" does not exist, skipping
CREATE TABLE tbl_on_heap (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision);
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 's1' as the Greenplum Database data distribution key for this table.
HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
INSERT INTO tbl_on_heap VALUES
('aaa','twoa','shpits','2011-06-01 12:30:30',23,732,834567,45.67,789.123,7.12345,123.456789),
('bbb','twob','shpits','2011-06-01 12:30:30',23,732,834567,45.67,789.123,7.12345,123.456789),
('ccc','twoc','shpits','2011-06-01 12:30:30',23,732,834567,45.67,789.123,7.12345,123.456789 );
DROP EXTERNAL TABLE IF EXISTS tbl;
NOTICE: table "tbl" does not exist, skipping
CREATE WRITABLE EXTERNAL TABLE tbl (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl2.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
INSERT INTO tbl SELECT * FROM tbl_on_heap;
SELECT * FROM tbl_on_heap ORDER BY s1;
s1 | s2 | s3 | dt | n1 | n2 | n3 | n4 | n5 | n6 | n7
-----+------+--------+--------------------------+----+-----+--------+-------+---------+---------+------------
aaa | twoa | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
bbb | twob | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
ccc | twoc | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
(3 rows)
-- gpfdist_ssl case 2
DROP TABLE IF EXISTS tbl_on_heap2;
NOTICE: table "tbl_on_heap2" does not exist, skipping
CREATE TABLE tbl_on_heap2 (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision);
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 's1' as the Greenplum Database data distribution key for this table.
HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
DROP EXTERNAL TABLE IF EXISTS tbl2;
NOTICE: table "tbl2" does not exist, skipping
CREATE EXTERNAL TABLE tbl2 (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl2.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
INSERT INTO tbl_on_heap2 SELECT * FROM tbl2;
SELECT * FROM tbl_on_heap2 ORDER BY s1;
s1 | s2 | s3 | dt | n1 | n2 | n3 | n4 | n5 | n6 | n7
-----+------+--------+--------------------------+----+-----+--------+-------+---------+---------+------------
aaa | twoa | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
bbb | twob | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
ccc | twoc | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
(3 rows)
-- gpfdist_ssl case 3
-- on same ETL server
DROP TABLE IF EXISTS tbl_on_heap;
CREATE TABLE tbl_on_heap (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision);
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 's1' as the Greenplum Database data distribution key for this table.
HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
DROP EXTERNAL TABLE IF EXISTS tbl;
CREATE EXTERNAL TABLE tbl (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl','gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl2.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
INSERT INTO tbl_on_heap SELECT * FROM tbl;
SELECT * FROM tbl_on_heap ORDER BY s1;
s1 | s2 | s3 | dt | n1 | n2 | n3 | n4 | n5 | n6 | n7
-----+------+--------+--------------------------+----+-----+--------+-------+---------+---------+------------
aaa | twoa | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
aaa | twoa | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
bbb | twob | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
bbb | twob | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
ccc | twoc | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
ccc | twoc | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
ddd | twod | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
eee | twoe | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
fff | twof | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
ggg | twog | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
hhh | twoh | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
iii | twoi | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
jjj | twoj | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
(13 rows)
-- gpfdist_ssl case 4
DROP TABLE IF EXISTS tbl_on_heap;
CREATE TABLE tbl_on_heap (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision);
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 's1' as the Greenplum Database data distribution key for this table.
HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
DROP EXTERNAL TABLE IF EXISTS tbl;
CREATE EXTERNAL TABLE tbl (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdist://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
INSERT INTO tbl_on_heap SELECT * FROM tbl;
ERROR: connection with gpfdist failed for gpfdist://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl. effective url: http://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl. error code = 104 (Connection reset by peer) (seg1 slice1 172.17.0.2:25433 pid=59561)
SELECT * FROM tbl_on_heap ORDER BY s1;
s1 | s2 | s3 | dt | n1 | n2 | n3 | n4 | n5 | n6 | n7
----+----+----+----+----+----+----+----+----+----+----
(0 rows)
-- gpfdist_ssl case 5
DROP TABLE IF EXISTS tbl_on_heap;
CREATE TABLE tbl_on_heap (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision);
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 's1' as the Greenplum Database data distribution key for this table.
HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
DROP EXTERNAL TABLE IF EXISTS tbl;
CREATE EXTERNAL TABLE tbl (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl','gpfdist://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
ERROR: URI protocols must be the same for all data sources
HINT: Available protocols are 'http', 'file', 'gpfdist' and 'gpfdists'
DROP TABLE IF EXISTS tbl_on_heap;
-- start_ignore
select * from gpfdist_ssl_stop;
x
-------------
stopping...
(1 row)
select * from gpfdist_ssl_not_matching_start;
x
-------------
starting...
(1 row)
-- end_ignore
-- gpfdist_ssl case 6
DROP EXTERNAL TABLE IF EXISTS tbl;
NOTICE: table "tbl" does not exist, skipping
CREATE EXTERNAL TABLE tbl (s1 text, s2 text, s3 text, dt timestamp,n1 smallint, n2 integer, n3 bigint, n4 decimal, n5 numeric, n6 real, n7 double precision)
LOCATION ('gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl')
FORMAT 'TEXT' (DELIMITER '|' );
SELECT * FROM tbl;
ERROR: connection with gpfdist failed for gpfdists://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl. effective url: https://127.0.0.1:7070/gpfdist_ssl/tbl1.tbl. (seg0 slice1 172.17.0.2:25432 pid=59537)
DETAIL: SSL certificate problem: self signed certificate
SET verify_gpfdists_cert=off;
WARNING: verify_gpfdists_cert=off. Greenplum Database will stop validating the gpfdists SSL certificate for connections between segments and gpfdists
SELECT * FROM tbl;
s1 | s2 | s3 | dt | n1 | n2 | n3 | n4 | n5 | n6 | n7
-----+------+--------+--------------------------+----+-----+--------+-------+---------+---------+------------
aaa | twoa | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
bbb | twob | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
ccc | twoc | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
ddd | twod | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
eee | twoe | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
fff | twof | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
ggg | twog | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
hhh | twoh | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
iii | twoi | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
jjj | twoj | shpits | Wed Jun 01 12:30:30 2011 | 23 | 732 | 834567 | 45.67 | 789.123 | 7.12345 | 123.456789
(10 rows)
-- start_ignore
select * from gpfdist_ssl_stop;
x
-------------
stopping...
(1 row)
-- end_ignore
src/bin/gpfdist/remote_regress/start_gpfdist_remote_win/start_gpfdist_ssl_not_matching.bash 0 → 100755
浏览文件 @ ae2a481a
#!/bin/bash -l
set -ex
nohup ssh -i "${REMOTE_KEY}" -T -L7070:127.0.0.1:7070 -p "${REMOTE_PORT}" "${REMOTE_USER}@${REMOTE_HOST}" \
"start_gpfdist_ssl_not_matching.bat" >/tmp/start_gpfdist_ssl_not_matching.log 2>&1 &
src/bin/gpfdist/remote_regress/start_gpfdist_remote_win/start_gpfdist_ssl_not_matching.bat 0 → 100644
浏览文件 @ ae2a481a
call "C:\Program Files\Greenplum\greenplum-clients\greenplum_clients_path.bat"
gpfdist -p 7070 -d .\ --ssl gpfdist_ssl\certs_not_matching
src/bin/gpfdist/remote_regress/start_gpfdist_remote_win/start_gpfdist_with_ssl.bash 0 → 100755
浏览文件 @ ae2a481a
#!/bin/bash -l
set -ex
nohup ssh -i "${REMOTE_KEY}" -T -L7070:127.0.0.1:7070 -p "${REMOTE_PORT}" "${REMOTE_USER}@${REMOTE_HOST}" \
"start_gpfdist_with_ssl.bat" >/tmp/start_gpfdist_with_ssl.log 2>&1 &
src/bin/gpfdist/remote_regress/start_gpfdist_remote_win/start_gpfdist_with_ssl.bat 0 → 100644
浏览文件 @ ae2a481a
call "C:\Program Files\Greenplum\greenplum-clients\greenplum_clients_path.bat"
del gpfdist_ssl\tbl2.tbl
gpfdist -p 7070 -d .\ --ssl gpfdist_ssl\certs_matching
src/bin/gpfdist/remote_regress/start_gpfdist_remote_win/stop_gpfdist.bash 0 → 100755
浏览文件 @ ae2a481a
#!/bin/bash -l
set -ex
ps -A -o pid,command | grep ssh |grep "start_gpfdist" |grep -v "gpdb" | awk '{print $1;}'|xargs kill
ssh -i "${REMOTE_KEY}" -T -p "${REMOTE_PORT}" "${REMOTE_USER}@${REMOTE_HOST}" \
"stop_gpfdist.bat" >/tmp/stop_gpfdist.log 2>&1
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或