From 693bcee5175b77eaa427076cc0edc93737228ed9 Mon Sep 17 00:00:00 2001 From: David Kimura Date: Thu, 13 Sep 2018 18:05:31 -0400 Subject: [PATCH] Add zstd as a (gp)contrib module After this patch: - zstd functions are no longer part of the built-in catalog - is only built when enabled with `--with-zstd` in autoconf Co-authored-by: Joao Pereira --- gpcontrib/Makefile | 14 +++-- gpcontrib/zstd/Makefile | 32 ++++++++++ .../zstd}/zstd_compression.c | 58 +++++-------------- gpcontrib/zstd/zstd_compression.sql | 22 +++++++ src/Makefile.global.in | 1 + src/backend/catalog/Makefile | 2 +- src/include/catalog/pg_compression.h | 2 - src/include/catalog/pg_proc.sql | 10 ---- src/include/catalog/pg_proc_gp.h | 20 ------- src/include/utils/builtins.h | 6 -- 10 files changed, 78 insertions(+), 89 deletions(-) create mode 100644 gpcontrib/zstd/Makefile rename {src/backend/catalog => gpcontrib/zstd}/zstd_compression.c (82%) create mode 100644 gpcontrib/zstd/zstd_compression.sql diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index 319cab1020..d906d2d371 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -16,12 +16,14 @@ include $(top_builddir)/src/Makefile.global # # targets built via top_builddir/GNUmakefile # - -$(call recurse,all install clean distclean,\ - gp_sparse_vector \ - gp_distribution_policy \ - gp_internal_tools \ - gp_inject_fault) +recurse_targets = gp_sparse_vector \ + gp_distribution_policy \ + gp_internal_tools \ + gp_inject_fault +ifeq "$(with_zstd)" "yes" + recurse_targets += zstd +endif +$(call recurse,all install clean distclean, $(recurse_targets)) all: gpcloud pxf mapreduce diff --git a/gpcontrib/zstd/Makefile b/gpcontrib/zstd/Makefile new file mode 100644 index 0000000000..0413441030 --- /dev/null +++ b/gpcontrib/zstd/Makefile @@ -0,0 +1,32 @@ +PG_CONFIG = pg_config + +MODULE_big = gp_zstd_compression +OBJS = zstd_compression.o +CFLAGS_SL += -lzstd +LDFLAGS_SL += -lzstd + +ifdef USE_PGXS + PGXS := $(shell pg_config --pgxs) + include $(PGXS) +else + top_builddir = ../.. + include $(top_builddir)/src/Makefile.global + include $(top_srcdir)/contrib/contrib-global.mk +endif + + +# Install into cdb_init.d, so that the catalog changes performed by initdb, +# and the compressor is available in all databases. +.PHONY: install-data +install-data: + $(INSTALL_DATA) zstd_compression.sql '$(DESTDIR)$(datadir)/cdb_init.d/zstd_compression.sql' + +install: install-data + +.PHONY: uninstall-data + +uninstall-data: + rm -f '$(DESTDIR)$(datadir)/cdb_init.d/zstd_compression.sql' + +uninstall: uninstall-data + diff --git a/src/backend/catalog/zstd_compression.c b/gpcontrib/zstd/zstd_compression.c similarity index 82% rename from src/backend/catalog/zstd_compression.c rename to gpcontrib/zstd/zstd_compression.c index 5cf6fd5383..b5acd67cad 100644 --- a/src/backend/catalog/zstd_compression.c +++ b/gpcontrib/zstd/zstd_compression.c @@ -15,11 +15,22 @@ #include "fmgr.h" #include "utils/builtins.h" -#ifdef HAVE_LIBZSTD -/* Zstandard library is provided */ - #include +Datum zstd_constructor(PG_FUNCTION_ARGS); +Datum zstd_destructor(PG_FUNCTION_ARGS); +Datum zstd_compress(PG_FUNCTION_ARGS); +Datum zstd_decompress(PG_FUNCTION_ARGS); +Datum zstd_validator(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(zstd_constructor); +PG_FUNCTION_INFO_V1(zstd_destructor); +PG_FUNCTION_INFO_V1(zstd_compress); +PG_FUNCTION_INFO_V1(zstd_decompress); +PG_FUNCTION_INFO_V1(zstd_validator); + +PG_MODULE_MAGIC; + /* Internal state for zstd */ typedef struct zstd_state { @@ -140,44 +151,3 @@ zstd_validator(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } - -#else /* HAVE_LIBZSTD */ -/* Zstandard library is not provided; use dummy functions instead */ - -#define NO_ZSTD_SUPPORT() \ - ereport(ERROR, \ - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \ - errmsg("Zstandard library is not supported by this build"), \ - errhint("Compile with --with-zstd to use Zstandard compression."))) - -Datum -zstd_constructor(PG_FUNCTION_ARGS) -{ - NO_ZSTD_SUPPORT(); -} - -Datum -zstd_destructor(PG_FUNCTION_ARGS) -{ - NO_ZSTD_SUPPORT(); -} - -Datum -zstd_compress(PG_FUNCTION_ARGS) -{ - NO_ZSTD_SUPPORT(); -} - -Datum -zstd_decompress(PG_FUNCTION_ARGS) -{ - NO_ZSTD_SUPPORT(); -} - -Datum -zstd_validator(PG_FUNCTION_ARGS) -{ - NO_ZSTD_SUPPORT(); -} - -#endif /* HAVE_LIBZSTD */ diff --git a/gpcontrib/zstd/zstd_compression.sql b/gpcontrib/zstd/zstd_compression.sql new file mode 100644 index 0000000000..d08ea99630 --- /dev/null +++ b/gpcontrib/zstd/zstd_compression.sql @@ -0,0 +1,22 @@ +CREATE FUNCTION gp_zstd_constructor(internal, internal, bool) RETURNS internal +LANGUAGE C VOLATILE AS '$libdir/gp_zstd_compression.so', 'zstd_constructor'; +COMMENT ON FUNCTION gp_zstd_constructor(internal, internal, bool) IS 'zstd compressor and decompressor constructor'; + +CREATE FUNCTION gp_zstd_destructor(internal) RETURNS void +LANGUAGE C VOLATILE AS '$libdir/gp_zstd_compression.so', 'zstd_destructor'; +COMMENT ON FUNCTION gp_zstd_destructor(internal) IS 'zstd compressor and decompressor destructor'; + +CREATE FUNCTION gp_zstd_compress(internal, int4, internal, int4, internal, internal) RETURNS void +LANGUAGE C VOLATILE AS '$libdir/gp_zstd_compression.so', 'zstd_compress'; +COMMENT ON FUNCTION gp_zstd_compress(internal, int4, internal, int4, internal, internal) IS 'zstd compressor'; + +CREATE FUNCTION gp_zstd_decompress(internal, int4, internal, int4, internal, internal) RETURNS void +LANGUAGE C VOLATILE AS '$libdir/gp_zstd_compression.so', 'zstd_decompress'; +COMMENT ON FUNCTION gp_zstd_decompress(internal, int4, internal, int4, internal, internal) IS 'zstd decompressor'; + +CREATE FUNCTION gp_zstd_validator(internal) RETURNS void +LANGUAGE C VOLATILE AS '$libdir/gp_zstd_compression.so', 'zstd_validator'; +COMMENT ON FUNCTION gp_zstd_validator(internal) IS 'zstdcompression validator'; + +INSERT INTO pg_catalog.pg_compression (compname, compconstructor, compdestructor, compcompressor, compdecompressor, compvalidator, compowner) +VALUES ('zstd', 'gp_zstd_constructor', 'gp_zstd_destructor', 'gp_zstd_compress', 'gp_zstd_decompress', 'gp_zstd_validator', 10 /* BOOSTRAP_SUPERUSERID */); diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 2eba765968..c505e539e7 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -224,6 +224,7 @@ PTHREAD_CFLAGS = @PTHREAD_CFLAGS@ PTHREAD_LIBS = @PTHREAD_LIBS@ have_yaml = @have_yaml@ +with_zstd = @with_zstd@ ########################################################################## diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 13f8e92904..0f2f12ee7a 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -26,7 +26,7 @@ OBJS += pg_exttable.o pg_extprotocol.o \ aoseg.o aoblkdir.o gp_fastsequence.o gp_segment_config.o \ pg_attribute_encoding.o pg_compression.o aovisimap.o \ pg_appendonly.o \ - oid_dispatch.o aocatalog.o zstd_compression.o $(QUICKLZ_COMPRESSION) + oid_dispatch.o aocatalog.o $(QUICKLZ_COMPRESSION) BKIFILES = postgres.bki postgres.description postgres.shdescription diff --git a/src/include/catalog/pg_compression.h b/src/include/catalog/pg_compression.h index 5e9d5b8031..0f56b2d6b4 100644 --- a/src/include/catalog/pg_compression.h +++ b/src/include/catalog/pg_compression.h @@ -77,8 +77,6 @@ DATA(insert OID = 7062 ( rle_type gp_rle_type_constructor gp_rle_type_destructor DATA(insert OID = 7063 ( none gp_dummy_compression_constructor gp_dummy_compression_destructor gp_dummy_compression_compress gp_dummy_compression_decompress gp_dummy_compression_validator PGUID )); -DATA(insert OID = 7070 ( zstd gp_zstd_constructor gp_zstd_destructor gp_zstd_compress gp_zstd_decompress gp_zstd_validator PGUID )); - #define NUM_COMPRESS_FUNCS 5 #define COMPRESSION_CONSTRUCTOR 0 diff --git a/src/include/catalog/pg_proc.sql b/src/include/catalog/pg_proc.sql index fc12a46ade..2d07a5a286 100644 --- a/src/include/catalog/pg_proc.sql +++ b/src/include/catalog/pg_proc.sql @@ -313,16 +313,6 @@ CREATE FUNCTION gp_rle_type_validator(internal) RETURNS void LANGUAGE internal IMMUTABLE AS 'rle_type_validator' WITH(OID=9923, DESCRIPTION="Type speific RLE compression validator"); - CREATE FUNCTION gp_zstd_constructor(internal, internal, bool) RETURNS internal LANGUAGE internal VOLATILE AS 'zstd_constructor' WITH (OID=7071, DESCRIPTION="zstd compressor and decompressor constructor"); - - CREATE FUNCTION gp_zstd_destructor(internal) RETURNS void LANGUAGE internal VOLATILE AS 'zstd_destructor' WITH(OID=7072, DESCRIPTION="zstd compressor and decompressor destructor"); - - CREATE FUNCTION gp_zstd_compress(internal, int4, internal, int4, internal, internal) RETURNS void LANGUAGE internal IMMUTABLE AS 'zstd_compress' WITH(OID=7073, DESCRIPTION="zstd compressor"); - - CREATE FUNCTION gp_zstd_decompress(internal, int4, internal, int4, internal, internal) RETURNS void LANGUAGE internal IMMUTABLE AS 'zstd_decompress' WITH(OID=7074, DESCRIPTION="zstd decompressor"); - - CREATE FUNCTION gp_zstd_validator(internal) RETURNS void LANGUAGE internal IMMUTABLE AS 'zstd_validator' WITH(OID=7075, DESCRIPTION="zstdcompression validator"); - CREATE FUNCTION gp_dummy_compression_constructor(internal, internal, bool) RETURNS internal LANGUAGE internal VOLATILE AS 'dummy_compression_constructor' WITH (OID=7064, DESCRIPTION="Dummy compression destructor"); CREATE FUNCTION gp_dummy_compression_destructor(internal) RETURNS internal LANGUAGE internal VOLATILE AS 'dummy_compression_destructor' WITH (OID=7065, DESCRIPTION="Dummy compression destructor"); diff --git a/src/include/catalog/pg_proc_gp.h b/src/include/catalog/pg_proc_gp.h index 485581455c..d23a95944b 100644 --- a/src/include/catalog/pg_proc_gp.h +++ b/src/include/catalog/pg_proc_gp.h @@ -631,26 +631,6 @@ DESCR("Type specific RLE decompressor"); DATA(insert OID = 9923 ( gp_rle_type_validator PGNSP PGUID 12 1 0 0 0 f f f f f f i 1 0 2278 "2281" _null_ _null_ _null_ _null_ rle_type_validator _null_ _null_ _null_ n a )); DESCR("Type speific RLE compression validator"); -/* gp_zstd_constructor(internal, internal, bool) => internal */ -DATA(insert OID = 7071 ( gp_zstd_constructor PGNSP PGUID 12 1 0 0 0 f f f f f f v 3 0 2281 "2281 2281 16" _null_ _null_ _null_ _null_ zstd_constructor _null_ _null_ _null_ n a )); -DESCR("zstd compressor and decompressor constructor"); - -/* gp_zstd_destructor(internal) => void */ -DATA(insert OID = 7072 ( gp_zstd_destructor PGNSP PGUID 12 1 0 0 0 f f f f f f v 1 0 2278 "2281" _null_ _null_ _null_ _null_ zstd_destructor _null_ _null_ _null_ n a )); -DESCR("zstd compressor and decompressor destructor"); - -/* gp_zstd_compress(internal, int4, internal, int4, internal, internal) => void */ -DATA(insert OID = 7073 ( gp_zstd_compress PGNSP PGUID 12 1 0 0 0 f f f f f f i 6 0 2278 "2281 23 2281 23 2281 2281" _null_ _null_ _null_ _null_ zstd_compress _null_ _null_ _null_ n a )); -DESCR("zstd compressor"); - -/* gp_zstd_decompress(internal, int4, internal, int4, internal, internal) => void */ -DATA(insert OID = 7074 ( gp_zstd_decompress PGNSP PGUID 12 1 0 0 0 f f f f f f i 6 0 2278 "2281 23 2281 23 2281 2281" _null_ _null_ _null_ _null_ zstd_decompress _null_ _null_ _null_ n a )); -DESCR("zstd decompressor"); - -/* gp_zstd_validator(internal) => void */ -DATA(insert OID = 7075 ( gp_zstd_validator PGNSP PGUID 12 1 0 0 0 f f f f f f i 1 0 2278 "2281" _null_ _null_ _null_ _null_ zstd_validator _null_ _null_ _null_ n a )); -DESCR("zstdcompression validator"); - /* gp_dummy_compression_constructor(internal, internal, bool) => internal */ DATA(insert OID = 7064 ( gp_dummy_compression_constructor PGNSP PGUID 12 1 0 0 0 f f f f f f v 3 0 2281 "2281 2281 16" _null_ _null_ _null_ _null_ dummy_compression_constructor _null_ _null_ _null_ n a )); DESCR("Dummy compression destructor"); diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index 79516c861a..01ef45c9b3 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -1347,12 +1347,6 @@ extern Datum rle_type_compress(PG_FUNCTION_ARGS); extern Datum rle_type_decompress(PG_FUNCTION_ARGS); extern Datum rle_type_validator(PG_FUNCTION_ARGS); -extern Datum zstd_constructor(PG_FUNCTION_ARGS); -extern Datum zstd_destructor(PG_FUNCTION_ARGS); -extern Datum zstd_compress(PG_FUNCTION_ARGS); -extern Datum zstd_decompress(PG_FUNCTION_ARGS); -extern Datum zstd_validator(PG_FUNCTION_ARGS); - extern Datum delta_constructor(PG_FUNCTION_ARGS); extern Datum delta_destructor(PG_FUNCTION_ARGS); extern Datum delta_compress(PG_FUNCTION_ARGS); -- GitLab