From c5d2bc315cee72484e35e34fc04ba666ba723e5a Mon Sep 17 00:00:00 2001 From: Haozhou Wang Date: Mon, 2 Jan 2017 22:37:20 -0500 Subject: [PATCH] Support 'OPTIONS' keyword in GPDB external table In case user wanna specify parameters with OPTIONS other than config file in LOCATION, it provides flexible and friendly grammar. Usage: CREATE EXTERNAL TABLE xxx LOCATION xxx FORMAT xxx [OPTIONS ( [option_key 'option_value' [, ...]] )] xxx; Signed-off-by: Kuien Liu Signed-off-by: Peifeng Qiu Signed-off-by: Adam Lee Signed-off-by: Yuan Zhao Signed-off-by: Jesse Zhang --- src/backend/catalog/pg_exttable.c | 115 ++++++++++++++++++ src/backend/commands/exttablecmds.c | 46 +++++++ src/backend/nodes/copyfuncs.c | 1 + src/backend/nodes/equalfuncs.c | 1 + src/backend/nodes/outfuncs.c | 1 + src/backend/nodes/readfast.c | 1 + src/backend/nodes/readfuncs.c | 1 + src/backend/parser/gram.y | 44 ++++++- src/bin/pg_dump/pg_dump.c | 34 +++++- src/bin/psql/describe.c | 19 ++- src/include/catalog/pg_exttable.h | 18 +-- src/include/catalog/pg_proc.sql | 4 +- src/include/catalog/pg_proc_gp.h | 8 +- src/include/nodes/parsenodes.h | 1 + src/include/parser/kwlist.h | 1 + src/test/regress/expected/dsp.out | 2 + src/test/regress/input/external_table.source | 21 ++++ src/test/regress/output/external_table.source | 39 ++++++ 18 files changed, 340 insertions(+), 17 deletions(-) diff --git a/src/backend/catalog/pg_exttable.c b/src/backend/catalog/pg_exttable.c index f388386e78..59b7e5a2ec 100644 --- a/src/backend/catalog/pg_exttable.c +++ b/src/backend/catalog/pg_exttable.c @@ -17,6 +17,7 @@ #include "catalog/pg_proc.h" #include "access/genam.h" #include "access/heapam.h" +#include "access/reloptions.h" #include "catalog/dependency.h" #include "catalog/indexing.h" #include "mb/pg_wchar.h" @@ -25,8 +26,12 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" #include "utils/fmgroids.h" +#include "utils/memutils.h" #include "utils/uri.h" +#include "miscadmin.h" +/* backport from FDW to support options */ +extern Datum pg_options_to_table(PG_FUNCTION_ARGS); /* * InsertExtTableEntry @@ -48,6 +53,7 @@ InsertExtTableEntry(Oid tbloid, Oid fmtErrTblOid, int encoding, Datum formatOptStr, + Datum optionsStr, Datum locationExec, Datum locationUris) { @@ -67,6 +73,7 @@ InsertExtTableEntry(Oid tbloid, values[Anum_pg_exttable_reloid - 1] = ObjectIdGetDatum(tbloid); values[Anum_pg_exttable_fmttype - 1] = CharGetDatum(formattype); values[Anum_pg_exttable_fmtopts - 1] = formatOptStr; + values[Anum_pg_exttable_options - 1] = optionsStr; if(commandString) { @@ -122,6 +129,84 @@ InsertExtTableEntry(Oid tbloid, heap_close(pg_exttable_rel, NoLock); } +/* + * deflist_to_tuplestore - Helper function to convert DefElem list to + * tuplestore usable in SRF. + */ +static void +deflist_to_tuplestore(ReturnSetInfo *rsinfo, List *options) +{ + ListCell *cell; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + Datum values[2]; + bool nulls[2]; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + + /* check to see if caller supports us returning a tuplestore */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize) || + rsinfo->expectedDesc == NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not allowed in this context"))); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + /* + * Now prepare the result set. + */ + tupdesc = CreateTupleDescCopy(rsinfo->expectedDesc); + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + foreach(cell, options) + { + DefElem *def = lfirst(cell); + + values[0] = CStringGetTextDatum(def->defname); + nulls[0] = false; + if (def->arg) + { + values[1] = CStringGetTextDatum(((Value *) (def->arg))->val.str); + nulls[1] = false; + } + else + { + values[1] = (Datum) 0; + nulls[1] = true; + } + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + + /* clean up and return the tuplestore */ + tuplestore_donestoring(tupstore); + + MemoryContextSwitchTo(oldcontext); +} + +/* + * Convert options array to name/value table. Useful for information + * schema and pg_dump. + */ +Datum +pg_options_to_table(PG_FUNCTION_ARGS) +{ + Datum array = PG_GETARG_DATUM(0); + + deflist_to_tuplestore((ReturnSetInfo *) fcinfo->resultinfo, + untransformRelOptions(array)); + + return (Datum) 0; +} + /* * Get the catalog entry for an exttable relation (from pg_exttable) */ @@ -154,6 +239,7 @@ GetExtTableEntryIfExists(Oid relid) Datum locations, fmtcode, fmtopts, + options, command, rejectlimit, rejectlimittype, @@ -261,7 +347,36 @@ GetExtTableEntryIfExists(Oid relid) Insist(!isNull); extentry->fmtopts = DatumGetCString(DirectFunctionCall1(textout, fmtopts)); + /* get the external table options string */ + options = heap_getattr(tuple, + Anum_pg_exttable_options, + RelationGetDescr(pg_exttable_rel), + &isNull); + if (isNull) + { + /* options list is always populated (url or ON X) */ + elog(ERROR, "could not find options for external protocol"); + } + else + { + Datum *elems; + int nelems; + int i; + char* option_str = NULL; + + deconstruct_array(DatumGetArrayTypeP(options), + TEXTOID, -1, false, 'i', + &elems, NULL, &nelems); + + for (i = 0; i < nelems; i++) + { + option_str = DatumGetCString(DirectFunctionCall1(textout, elems[i])); + + /* append to a list of Value nodes, size nelems */ + extentry->options = lappend(extentry->options, makeString(pstrdup(option_str))); + } + } /* get the reject limit */ rejectlimit = heap_getattr(tuple, diff --git a/src/backend/commands/exttablecmds.c b/src/backend/commands/exttablecmds.c index cd63a2858e..cad79c6108 100644 --- a/src/backend/commands/exttablecmds.c +++ b/src/backend/commands/exttablecmds.c @@ -40,6 +40,7 @@ static Datum transformExecOnClause(List *on_clause); static char transformFormatType(char *formatname); static Datum transformFormatOpts(char formattype, List *formatOpts, int numcols, bool iswritable); static void InvokeProtocolValidation(Oid procOid, char *procName, bool iswritable, List *locs); +static Datum optionsListToArray(List *options); /* ---------------------------------------------------------------- * DefineExternalRelation @@ -67,6 +68,7 @@ DefineExternalRelation(CreateExternalStmt *createExtStmt) Oid reloid = 0; Oid fmtErrTblOid = InvalidOid; Datum formatOptStr; + Datum optionsStr; Datum locationUris = 0; Datum locationExec = 0; char *commandString = NULL; @@ -292,6 +294,14 @@ DefineExternalRelation(CreateExternalStmt *createExtStmt) list_length(createExtStmt->tableElts), iswritable); + /* + * Parse and validate OPTION clause. + */ + optionsStr = optionsListToArray(createExtStmt->extOptions); + if (DatumGetPointer(optionsStr) == NULL) + { + optionsStr = PointerGetDatum(construct_empty_array(TEXTOID)); + } /* * Parse single row error handling info if available */ @@ -401,6 +411,7 @@ DefineExternalRelation(CreateExternalStmt *createExtStmt) fmtErrTblOid, encoding, formatOptStr, + optionsStr, locationExec, locationUris); @@ -761,6 +772,41 @@ transformFormatType(char *formatname) return result; } +/* + * Transform the external table options into a text array format. + * + * The result is an array that includes the format string. + * + * This method is a backported FDW's function from upper stream . + */ +static Datum +optionsListToArray(List *options) +{ + ArrayBuildState *astate = NULL; + ListCell *option; + + foreach(option, options) + { + DefElem *defel = (DefElem *) lfirst(option); + char *key = defel->defname; + char *val = defGetString(defel); + text *t; + Size len; + + // first 1 for '=', last 1 for '\0' + len = VARHDRSZ + strlen(key) + 1 + strlen(val) + 1; + t = palloc(len); + SET_VARSIZE(t, len); + sprintf(VARDATA(t), "%s=%s", key, val); + astate = accumArrayResult(astate, PointerGetDatum(t), false, + TEXTOID, CurrentMemoryContext); + } + + if (astate) + return makeArrayResult(astate, CurrentMemoryContext); + + return PointerGetDatum(NULL); +} /* * Transform the FORMAT options into a text field. Parse the diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index aa3a985509..ccc59ec147 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -3291,6 +3291,7 @@ _copyCreateExternalStmt(CreateExternalStmt *from) COPY_SCALAR_FIELD(isweb); COPY_SCALAR_FIELD(iswritable); COPY_NODE_FIELD(sreh); + COPY_NODE_FIELD(extOptions); COPY_NODE_FIELD(encoding); COPY_NODE_FIELD(distributedBy); if (from->policy) diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 2c67e7525d..318f8edc50 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1224,6 +1224,7 @@ _equalCreateExternalStmt(CreateExternalStmt *a, CreateExternalStmt *b) COMPARE_SCALAR_FIELD(isweb); COMPARE_SCALAR_FIELD(iswritable); COMPARE_NODE_FIELD(sreh); + COMPARE_NODE_FIELD(extOptions); COMPARE_NODE_FIELD(encoding); COMPARE_NODE_FIELD(distributedBy); diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 8c7bfc133a..5676094d6a 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2322,6 +2322,7 @@ _outCreateExternalStmt(StringInfo str, CreateExternalStmt *node) WRITE_BOOL_FIELD(isweb); WRITE_BOOL_FIELD(iswritable); WRITE_NODE_FIELD(sreh); + WRITE_NODE_FIELD(extOptions); WRITE_NODE_FIELD(encoding); WRITE_NODE_FIELD(distributedBy); } diff --git a/src/backend/nodes/readfast.c b/src/backend/nodes/readfast.c index 9e037406a5..0999110167 100644 --- a/src/backend/nodes/readfast.c +++ b/src/backend/nodes/readfast.c @@ -1304,6 +1304,7 @@ _readCreateExternalStmt(void) READ_BOOL_FIELD(isweb); READ_BOOL_FIELD(iswritable); READ_NODE_FIELD(sreh); + READ_NODE_FIELD(extOptions); READ_NODE_FIELD(encoding); READ_NODE_FIELD(distributedBy); diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index ef30ded5f6..613f936ca1 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -2244,6 +2244,7 @@ _readCreateExternalStmt(void) READ_BOOL_FIELD(isweb); READ_BOOL_FIELD(iswritable); READ_NODE_FIELD(sreh); + READ_NODE_FIELD(extOptions); READ_NODE_FIELD(encoding); READ_NODE_FIELD(distributedBy); local_node->policy = NULL; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 218cc09fb6..a30072d71a 100755 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -227,9 +227,11 @@ static Node *makeIsNotDistinctFromNode(Node *expr, int position); %type createdb_opt_list alterdb_opt_list copy_opt_list ext_on_clause_list format_opt format_opt_list format_def_list transaction_mode_list + ext_options ext_options_opt ext_options_list ext_opt_encoding_list create_extension_opt_list alter_extension_opt_list %type createdb_opt_item alterdb_opt_item copy_opt_item ext_on_clause_item format_opt_item format_def_item transaction_mode_item + ext_options_item ext_opt_encoding_item create_extension_opt_item alter_extension_opt_item %type opt_lock lock_type cast_context @@ -528,7 +530,7 @@ static Node *makeIsNotDistinctFromNode(Node *expr, int position); NOCREATEROLE NOCREATEUSER NOINHERIT NOLOGIN_P NONE NOSUPERUSER NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF NULLS_P NUMERIC - OBJECT_P OF OFF OFFSET OIDS OLD ON ONLY OPERATOR OPTION OR + OBJECT_P OF OFF OFFSET OIDS OLD ON ONLY OPERATOR OPTION OPTIONS OR ORDER OUT_P OUTER_P OVERLAPS OVERLAY OWNED OWNER PARSER PARTIAL PASSWORD PLACING PLANS POSITION @@ -809,6 +811,7 @@ static Node *makeIsNotDistinctFromNode(Node *expr, int position); %nonassoc OF %nonassoc OIDS %nonassoc OPTION + %nonassoc OPTIONS %nonassoc OTHERS %nonassoc OVER %nonassoc OVERCOMMIT @@ -4604,7 +4607,7 @@ opt_with_data: *****************************************************************************/ CreateExternalStmt: CREATE OptWritable EXTERNAL OptWeb OptTemp TABLE qualified_name '(' OptExtTableElementList ')' - ExtTypedesc FORMAT Sconst format_opt ext_opt_encoding_list OptSingleRowErrorHandling OptDistributedBy + ExtTypedesc FORMAT Sconst format_opt ext_options_opt ext_opt_encoding_list OptSingleRowErrorHandling OptDistributedBy { CreateExternalStmt *n = makeNode(CreateExternalStmt); n->iswritable = $2; @@ -4615,9 +4618,10 @@ CreateExternalStmt: CREATE OptWritable EXTERNAL OptWeb OptTemp TABLE qualified_n n->exttypedesc = (ExtTableTypeDesc *) $11; n->format = $13; n->formatOpts = $14; - n->encoding = $15; - n->sreh = $16; - n->distributedBy = $17; + n->extOptions = $15; + n->encoding = $16; + n->sreh = $17; + n->distributedBy = $18; n->policy = 0; /* various syntax checks for EXECUTE external table */ @@ -4800,6 +4804,34 @@ format_opt_item: } ; +ext_options_opt: + OPTIONS ext_options { $$ = $2; } + | /*EMPTY*/ { $$ = NIL; } + ; + +ext_options: + '(' ext_options_list ')' { $$ = $2; } + | '(' ')' { $$ = NIL; } + ; + +ext_options_list: + ext_options_item + { + $$ = list_make1($1); + } + | ext_options_list ',' ext_options_item + { + $$ = lappend($1, $3); + } + ; + +ext_options_item: + ColLabel Sconst + { + $$ = makeDefElem($1, (Node *)makeString($2)); + } + ; + OptExtTableElementList: ExtTableElementList { $$ = $1; } | /*EMPTY*/ { $$ = NIL; } @@ -13031,6 +13063,7 @@ unreserved_keyword: | OIDS | OPERATOR | OPTION + | OPTIONS | ORDERED | OTHERS | OVER @@ -13318,6 +13351,7 @@ PartitionIdentKeyword: ABORT_P | OIDS | OPERATOR | OPTION + | OPTIONS | OTHERS | OVERCOMMIT | OWNED diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 903628dd2e..50f1a2fefa 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -10084,6 +10084,8 @@ dumpExternal(TableInfo *tbinfo, PQExpBuffer query, PQExpBuffer q, PQExpBuffer de char *customfmt = NULL; bool isweb = false; bool iswritable = false; + char *options; + bool gpdb5 = isGPDB5000OrLater(); /* * DROP must be fully qualified in case same name appears in @@ -10095,7 +10097,27 @@ dumpExternal(TableInfo *tbinfo, PQExpBuffer query, PQExpBuffer q, PQExpBuffer de fmtId(tbinfo->dobj.name)); /* Now get required information from pg_exttable */ - if (g_fout->remoteVersion >= 80214) + if (gpdb5) + { + appendPQExpBuffer(query, + "SELECT x.location, x.fmttype, x.fmtopts, x.command, " + "x.rejectlimit, x.rejectlimittype, " + "(SELECT relname " + "FROM pg_catalog.pg_class " + "WHERE Oid=x.fmterrtbl) AS errtblname, " + "x.fmterrtbl = x.reloid AS errortofile , " + "pg_catalog.pg_encoding_to_char(x.encoding), " + "x.writable, " + "array_to_string(ARRAY( " + "SELECT pg_catalog.quote_ident(option_name) || ' ' || " + "pg_catalog.quote_literal(option_value) " + "FROM pg_options_to_table(x.options) " + "ORDER BY option_name" + "), E',\n ') AS options " + "FROM pg_catalog.pg_exttable x, pg_catalog.pg_class c " + "WHERE x.reloid = c.oid AND c.oid = '%u'::oid ", tbinfo->dobj.catId.oid); + } + else if (g_fout->remoteVersion >= 80214) { appendPQExpBuffer(query, "SELECT x.location, x.fmttype, x.fmtopts, x.command, " @@ -10165,6 +10187,11 @@ dumpExternal(TableInfo *tbinfo, PQExpBuffer query, PQExpBuffer q, PQExpBuffer de extencoding = PQgetvalue(res, 0, 8); writable = PQgetvalue(res, 0, 9); + if (gpdb5) + { + options = PQgetvalue(res, 0, 10); + } + if ((command && strlen(command) > 0) || (strncmp(locations + 1, "http", strlen("http")) == 0)) isweb = true; @@ -10292,6 +10319,11 @@ dumpExternal(TableInfo *tbinfo, PQExpBuffer query, PQExpBuffer q, PQExpBuffer de customfmt = NULL; } + if (gpdb5) + { + appendPQExpBuffer(q, "OPTIONS (\n %s\n )\n", options); + } + if (g_fout->remoteVersion >= 80205) { /* add ENCODING clause */ diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 7702c5e2cb..970afa3322 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -1805,6 +1805,8 @@ describeOneTableDetails(const char *schemaname, { /* Footer information about an external table */ PGresult *result; + bool gpdb5 = isGPDB5000OrLater(); + char *optionsName = gpdb5 ? ", x.options " : ""; printfPQExpBuffer(&buf, "SELECT x.location, x.fmttype, x.fmtopts, x.command, " @@ -1814,8 +1816,9 @@ describeOneTableDetails(const char *schemaname, "WHERE Oid=x.fmterrtbl) AS errtblname, " "pg_catalog.pg_encoding_to_char(x.encoding), " "x.fmterrtbl = x.reloid AS errortofile " + "%s" "FROM pg_catalog.pg_exttable x, pg_catalog.pg_class c " - "WHERE x.reloid = c.oid AND c.oid = '%s'\n", oid); + "WHERE x.reloid = c.oid AND c.oid = '%s'\n", optionsName, oid); result = PSQLexec(buf.data, false); if (!result) @@ -1838,6 +1841,11 @@ describeOneTableDetails(const char *schemaname, char *extencoding = PQgetvalue(result, 0, 8); char *errortofile = PQgetvalue(result, 0, 9); char *format; + char *options; + if (gpdb5) + { + options = PQgetvalue(result, 0, 10); + } /* Writable/Readable */ printfPQExpBuffer(&tmpbuf, _("Type: %s"), writable[0] == 't' ? "writable" : "readable"); @@ -1889,6 +1897,13 @@ describeOneTableDetails(const char *schemaname, printfPQExpBuffer(&tmpbuf, _("Format options: %s"), fmtopts); printTableAddFooter(&cont, tmpbuf.data); + if (gpdb5) + { + /* external table options */ + printfPQExpBuffer(&tmpbuf, _("External options: %s"), options); + printTableAddFooter(&cont, tmpbuf.data); + } + if(command && strlen(command) > 0) { /* EXECUTE type table - show command and command location */ @@ -4228,4 +4243,4 @@ printACLColumn(PQExpBuffer buf, const char *colname) appendPQExpBuffer(buf, "pg_catalog.array_to_string(%s, '\\n') AS \"%s\"", colname, gettext_noop("Access privileges")); -} \ No newline at end of file +} diff --git a/src/include/catalog/pg_exttable.h b/src/include/catalog/pg_exttable.h index a415db8e28..244cecf1e9 100644 --- a/src/include/catalog/pg_exttable.h +++ b/src/include/catalog/pg_exttable.h @@ -31,6 +31,7 @@ CATALOG(pg_exttable,6040) BKI_WITHOUT_OIDS text location[1]; /* array of URI strings */ char fmttype; /* 't' (text) or 'c' (csv) */ text fmtopts; /* the data format options */ + text options[1]; /* the array of external table options */ text command; /* the command string to EXECUTE */ int4 rejectlimit; /* error count reject limit per segment */ char rejectlimittype; /* 'r' (rows) or 'p' (percent) */ @@ -55,17 +56,18 @@ typedef FormData_pg_exttable *Form_pg_exttable; * compiler constants for pg_exttable * ---------------- */ -#define Natts_pg_exttable 10 +#define Natts_pg_exttable 11 #define Anum_pg_exttable_reloid 1 #define Anum_pg_exttable_location 2 #define Anum_pg_exttable_fmttype 3 #define Anum_pg_exttable_fmtopts 4 -#define Anum_pg_exttable_command 5 -#define Anum_pg_exttable_rejectlimit 6 -#define Anum_pg_exttable_rejectlimittype 7 -#define Anum_pg_exttable_fmterrtbl 8 -#define Anum_pg_exttable_encoding 9 -#define Anum_pg_exttable_writable 10 +#define Anum_pg_exttable_options 5 +#define Anum_pg_exttable_command 6 +#define Anum_pg_exttable_rejectlimit 7 +#define Anum_pg_exttable_rejectlimittype 8 +#define Anum_pg_exttable_fmterrtbl 9 +#define Anum_pg_exttable_encoding 10 +#define Anum_pg_exttable_writable 11 /* @@ -77,6 +79,7 @@ typedef struct ExtTableEntry List* locations; char fmtcode; char* fmtopts; + List* options; char* command; int rejectlimit; char rejectlimittype; @@ -99,6 +102,7 @@ extern void InsertExtTableEntry(Oid tbloid, Oid fmtErrTblOid, int encoding, Datum formatOptStr, + Datum optionsStr, Datum locationExec, Datum locationUris); diff --git a/src/include/catalog/pg_proc.sql b/src/include/catalog/pg_proc.sql index 18dc5da8c1..52f109968c 100644 --- a/src/include/catalog/pg_proc.sql +++ b/src/include/catalog/pg_proc.sql @@ -1861,4 +1861,6 @@ CREATE FUNCTION gp_nondbspecific_ptcat_verification() RETURNS bool LANGUAGE inte CREATE FUNCTION complex_lte(complex, complex) RETURNS bool LANGUAGE internal IMMUTABLE STRICT AS 'complex_lte' WITH (OID=3595, DESCRIPTION="less than or equal"); CREATE FUNCTION complex_gte(complex, complex) RETURNS bool LANGUAGE internal IMMUTABLE STRICT AS 'complex_gte' WITH (OID=3596, DESCRIPTION="greater than or equal"); - + + -- functions for external table + CREATE FUNCTION pg_options_to_table(IN options_array _text, OUT option_name text, OUT option_value text) RETURNS SETOF pg_catalog.record LANGUAGE internal IMMUTABLE STRICT AS 'pg_options_to_table' WITH (OID=2022, DESCRIPTION="convert generic options array to name/value table"); diff --git a/src/include/catalog/pg_proc_gp.h b/src/include/catalog/pg_proc_gp.h index 4d174eb43c..c9274db4f7 100644 --- a/src/include/catalog/pg_proc_gp.h +++ b/src/include/catalog/pg_proc_gp.h @@ -22,7 +22,7 @@ WARNING: DO NOT MODIFY THE FOLLOWING SECTION: Generated by catullus.pl version 8 - on Tue Nov 15 14:46:37 2016 + on Thu Dec 29 06:06:55 2016 Please make your changes in pg_proc.sql */ @@ -3116,4 +3116,10 @@ DATA(insert OID = 3596 ( complex_gte PGNSP PGUID 12 1 0 0 f f t f i 2 0 16 f "1 DESCR("greater than or equal"); + /* functions for external table */ +/* pg_options_to_table(IN options_array _text, OUT option_name text, OUT option_value text) => SETOF pg_catalog.record */ +DATA(insert OID = 2022 ( pg_options_to_table PGNSP PGUID 12 1 1000 0 f f t t i 1 0 2249 f "1009" "{1009,25,25}" "{i,o,o}" "{options_array,option_name,option_value}" _null_ pg_options_to_table _null_ _null_ _null_ n )); +DESCR("convert generic options array to name/value table"); + + /* TIDYCAT_END_PG_PROC_GEN */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index a083290af1..7425b8194f 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1545,6 +1545,7 @@ typedef struct CreateExternalStmt bool isweb; bool iswritable; Node *sreh; /* Single row error handling info */ + List *extOptions; /* generic options to external table */ List *encoding; /* List (size 1 max) of DefElem nodes for data encoding */ List *distributedBy; /* what columns we distribute the data by */ diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index e9a2f1f6bd..60b17bc3a5 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -302,6 +302,7 @@ PG_KEYWORD("on", ON, RESERVED_KEYWORD) PG_KEYWORD("only", ONLY, RESERVED_KEYWORD) PG_KEYWORD("operator", OPERATOR, UNRESERVED_KEYWORD) PG_KEYWORD("option", OPTION, UNRESERVED_KEYWORD) +PG_KEYWORD("options", OPTIONS, UNRESERVED_KEYWORD) PG_KEYWORD("or", OR, RESERVED_KEYWORD) PG_KEYWORD("order", ORDER, RESERVED_KEYWORD) PG_KEYWORD("ordered", ORDERED, UNRESERVED_KEYWORD) diff --git a/src/test/regress/expected/dsp.out b/src/test/regress/expected/dsp.out index 7ed09f2d05..8a60467483 100644 --- a/src/test/regress/expected/dsp.out +++ b/src/test/regress/expected/dsp.out @@ -810,6 +810,7 @@ Type: readable Encoding: UTF8 Format type: text Format options: delimiter ' ' null '\N' escape '\' +External options: {} External location: file:///tmp/test.txt create external table ext_t2 (a int, b int) @@ -825,6 +826,7 @@ Type: readable Encoding: UTF8 Format type: text Format options: delimiter ' ' null '\N' escape '\' +External options: {} External location: file:///tmp/test.txt Segment reject limit: 100 rows Error Log in File diff --git a/src/test/regress/input/external_table.source b/src/test/regress/input/external_table.source index 066d58c443..669c9d67e1 100644 --- a/src/test/regress/input/external_table.source +++ b/src/test/regress/input/external_table.source @@ -1358,6 +1358,27 @@ SELECT COUNT(*) FROM gp_read_error_log('exttab_heap_join_1'); \! rm @abs_srcdir@/data/tableless.csv +-- Create external table with 'OPTIONS' +CREATE EXTERNAL TABLE exttab_with_option_empty( i int, j text ) +LOCATION ('file://@hostname@@abs_srcdir@/data/exttab_few_errors.data') FORMAT 'TEXT' (DELIMITER '|') +OPTIONS (); + +CREATE EXTERNAL TABLE exttab_with_option_1( i int, j text ) +LOCATION ('file://@hostname@@abs_srcdir@/data/exttab_few_errors.data') FORMAT 'TEXT' (DELIMITER '|') +OPTIONS (hello 'world'); + +CREATE EXTERNAL TABLE exttab_with_options( i int, j text ) +LOCATION ('file://@hostname@@abs_srcdir@/data/exttab_few_errors.data') FORMAT 'TEXT' (DELIMITER '|') +OPTIONS (hello 'world', bonjour 'again', nihao 'again and again' ); + +\d exttab_with_options + +\d exttab_with_option_empty + +DROP EXTERNAL TABLE IF EXISTS exttab_with_option_empty; +DROP EXTERNAL TABLE IF EXISTS exttab_with_option_1; +DROP EXTERNAL TABLE IF EXISTS exttab_with_options; + -- start_ignore -- drop temp external protocols DROP PROTOCOL if exists demoprot; diff --git a/src/test/regress/output/external_table.source b/src/test/regress/output/external_table.source index 639cd76a9c..b5c4fa317b 100644 --- a/src/test/regress/output/external_table.source +++ b/src/test/regress/output/external_table.source @@ -2781,6 +2781,45 @@ SELECT COUNT(*) FROM gp_read_error_log('exttab_heap_join_1'); (1 row) \! rm @abs_srcdir@/data/tableless.csv +-- Create external table with 'OPTIONS' +CREATE EXTERNAL TABLE exttab_with_option_empty( i int, j text ) +LOCATION ('file://@hostname@@abs_srcdir@/data/exttab_few_errors.data') FORMAT 'TEXT' (DELIMITER '|') +OPTIONS (); +CREATE EXTERNAL TABLE exttab_with_option_1( i int, j text ) +LOCATION ('file://@hostname@@abs_srcdir@/data/exttab_few_errors.data') FORMAT 'TEXT' (DELIMITER '|') +OPTIONS (hello 'world'); +CREATE EXTERNAL TABLE exttab_with_options( i int, j text ) +LOCATION ('file://@hostname@@abs_srcdir@/data/exttab_few_errors.data') FORMAT 'TEXT' (DELIMITER '|') +OPTIONS (hello 'world', bonjour 'again', nihao 'again and again' ); +\d exttab_with_options +External table "public.exttab_with_options" + Column | Type | Modifiers +--------+---------+----------- + i | integer | + j | text | +Type: readable +Encoding: UTF8 +Format type: text +Format options: delimiter '|' null '\N' escape '\' +External options: {hello=world,bonjour=again,"nihao=again and again"} +External location: file://@hostname@@abs_srcdir@/data/exttab_few_errors.data + +\d exttab_with_option_empty +External table "public.exttab_with_option_empty" + Column | Type | Modifiers +--------+---------+----------- + i | integer | + j | text | +Type: readable +Encoding: UTF8 +Format type: text +Format options: delimiter '|' null '\N' escape '\' +External options: {} +External location: file://@hostname@@abs_srcdir@/data/exttab_few_errors.data + +DROP EXTERNAL TABLE IF EXISTS exttab_with_option_empty; +DROP EXTERNAL TABLE IF EXISTS exttab_with_option_1; +DROP EXTERNAL TABLE IF EXISTS exttab_with_options; -- start_ignore -- drop temp external protocols DROP PROTOCOL if exists demoprot; -- GitLab