Add zstd as a (gp)contrib module

After this patch:

- zstd functions are no longer part of the built-in catalog
- is only built when enabled with `--with-zstd` in autoconf
Co-authored-by: NJoao Pereira <jdealmeidapereira@pivotal.io>

gpcontrib/Makefile

@@ -16,12 +16,14 @@ include $(top_builddir)/src/Makefile.global
 #
 # targets built via top_builddir/GNUmakefile
 #
-
-$(call recurse,all install clean distclean,\
-			   gp_sparse_vector \
-			   gp_distribution_policy \
-			   gp_internal_tools \
-			   gp_inject_fault)
+recurse_targets = gp_sparse_vector \
+               gp_distribution_policy \
+               gp_internal_tools \
+               gp_inject_fault
+ifeq "$(with_zstd)" "yes"
+	recurse_targets += zstd
+endif
+$(call recurse,all install clean distclean, $(recurse_targets))
 
 all: gpcloud pxf mapreduce
 

gpcontrib/zstd/Makefile

+PG_CONFIG = pg_config
+
+MODULE_big = gp_zstd_compression
+OBJS = zstd_compression.o
+CFLAGS_SL += -lzstd
+LDFLAGS_SL += -lzstd
+
+ifdef USE_PGXS
+  PGXS := $(shell pg_config --pgxs)
+  include $(PGXS)
+else
+  top_builddir = ../..
+  include $(top_builddir)/src/Makefile.global
+  include $(top_srcdir)/contrib/contrib-global.mk
+endif
+
+
+# Install into cdb_init.d, so that the catalog changes performed by initdb,
+# and the compressor is available in all databases.
+.PHONY: install-data
+install-data:
+	$(INSTALL_DATA) zstd_compression.sql '$(DESTDIR)$(datadir)/cdb_init.d/zstd_compression.sql'
+
+install: install-data
+
+.PHONY: uninstall-data
+
+uninstall-data:
+	rm -f '$(DESTDIR)$(datadir)/cdb_init.d/zstd_compression.sql'
+
+uninstall: uninstall-data
+

src/backend/catalog/zstd_compression.c → gpcontrib/zstd/zstd_compression.c

@@ -15,11 +15,22 @@
 #include "fmgr.h"
 #include "utils/builtins.h"
 
-#ifdef HAVE_LIBZSTD
-/* Zstandard library is provided */
-
 #include <zstd.h>
 
+Datum zstd_constructor(PG_FUNCTION_ARGS);
+Datum zstd_destructor(PG_FUNCTION_ARGS);
+Datum zstd_compress(PG_FUNCTION_ARGS);
+Datum zstd_decompress(PG_FUNCTION_ARGS);
+Datum zstd_validator(PG_FUNCTION_ARGS);
+
+PG_FUNCTION_INFO_V1(zstd_constructor);
+PG_FUNCTION_INFO_V1(zstd_destructor);
+PG_FUNCTION_INFO_V1(zstd_compress);
+PG_FUNCTION_INFO_V1(zstd_decompress);
+PG_FUNCTION_INFO_V1(zstd_validator);
+
+PG_MODULE_MAGIC;
+
 /* Internal state for zstd */
 typedef struct zstd_state
 {
@@ -140,44 +151,3 @@ zstd_validator(PG_FUNCTION_ARGS)
 	PG_RETURN_VOID();
 }
 
-
-#else							/* HAVE_LIBZSTD */
-/* Zstandard library is not provided; use dummy functions instead */
-
-#define NO_ZSTD_SUPPORT() \
-	ereport(ERROR, \
-			(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
-			 errmsg("Zstandard library is not supported by this build"), \
-			 errhint("Compile with --with-zstd to use Zstandard compression.")))
-
-Datum
-zstd_constructor(PG_FUNCTION_ARGS)
-{
-	NO_ZSTD_SUPPORT();
-}
-
-Datum
-zstd_destructor(PG_FUNCTION_ARGS)
-{
-	NO_ZSTD_SUPPORT();
-}
-
-Datum
-zstd_compress(PG_FUNCTION_ARGS)
-{
-	NO_ZSTD_SUPPORT();
-}
-
-Datum
-zstd_decompress(PG_FUNCTION_ARGS)
-{
-	NO_ZSTD_SUPPORT();
-}
-
-Datum
-zstd_validator(PG_FUNCTION_ARGS)
-{
-	NO_ZSTD_SUPPORT();
-}
-
-#endif							/* HAVE_LIBZSTD */

gpcontrib/zstd/zstd_compression.sql

+CREATE FUNCTION gp_zstd_constructor(internal, internal, bool) RETURNS internal
+LANGUAGE C VOLATILE AS '$libdir/gp_zstd_compression.so', 'zstd_constructor';
+COMMENT ON FUNCTION gp_zstd_constructor(internal, internal, bool) IS 'zstd compressor and decompressor constructor';
+
+CREATE FUNCTION gp_zstd_destructor(internal) RETURNS void
+LANGUAGE C VOLATILE AS '$libdir/gp_zstd_compression.so', 'zstd_destructor';
+COMMENT ON FUNCTION gp_zstd_destructor(internal) IS 'zstd compressor and decompressor destructor';
+
+CREATE FUNCTION gp_zstd_compress(internal, int4, internal, int4, internal, internal) RETURNS void
+LANGUAGE C VOLATILE AS '$libdir/gp_zstd_compression.so', 'zstd_compress';
+COMMENT ON FUNCTION gp_zstd_compress(internal, int4, internal, int4, internal, internal) IS 'zstd compressor';
+
+CREATE FUNCTION gp_zstd_decompress(internal, int4, internal, int4, internal, internal) RETURNS void
+LANGUAGE C VOLATILE AS '$libdir/gp_zstd_compression.so', 'zstd_decompress';
+COMMENT ON FUNCTION gp_zstd_decompress(internal, int4, internal, int4, internal, internal) IS 'zstd decompressor';
+
+CREATE FUNCTION gp_zstd_validator(internal) RETURNS void
+LANGUAGE C VOLATILE AS '$libdir/gp_zstd_compression.so', 'zstd_validator';
+COMMENT ON FUNCTION gp_zstd_validator(internal) IS 'zstdcompression validator';
+
+INSERT INTO pg_catalog.pg_compression (compname, compconstructor, compdestructor, compcompressor, compdecompressor, compvalidator, compowner)
+VALUES ('zstd', 'gp_zstd_constructor', 'gp_zstd_destructor', 'gp_zstd_compress', 'gp_zstd_decompress', 'gp_zstd_validator', 10 /* BOOSTRAP_SUPERUSERID */);

src/Makefile.global.in

@@ -224,6 +224,7 @@ PTHREAD_CFLAGS		= @PTHREAD_CFLAGS@
 PTHREAD_LIBS		= @PTHREAD_LIBS@
 
 have_yaml 		= @have_yaml@
+with_zstd = @with_zstd@
 
 
 ##########################################################################

src/backend/catalog/Makefile

@@ -26,7 +26,7 @@ OBJS += pg_exttable.o pg_extprotocol.o \
        aoseg.o aoblkdir.o gp_fastsequence.o gp_segment_config.o \
        pg_attribute_encoding.o pg_compression.o aovisimap.o \
        pg_appendonly.o \
-       oid_dispatch.o aocatalog.o zstd_compression.o $(QUICKLZ_COMPRESSION)
+       oid_dispatch.o aocatalog.o $(QUICKLZ_COMPRESSION)
 
 BKIFILES = postgres.bki postgres.description postgres.shdescription
 

src/include/catalog/pg_compression.h

@@ -77,8 +77,6 @@ DATA(insert OID = 7062 ( rle_type gp_rle_type_constructor gp_rle_type_destructor
 
 DATA(insert OID = 7063 ( none gp_dummy_compression_constructor gp_dummy_compression_destructor gp_dummy_compression_compress gp_dummy_compression_decompress gp_dummy_compression_validator PGUID ));
 
-DATA(insert OID = 7070 ( zstd gp_zstd_constructor gp_zstd_destructor gp_zstd_compress gp_zstd_decompress gp_zstd_validator PGUID ));
-
 #define NUM_COMPRESS_FUNCS 5
 
 #define COMPRESSION_CONSTRUCTOR 0

src/include/catalog/pg_proc.sql

@@ -313,16 +313,6 @@
 
  CREATE FUNCTION gp_rle_type_validator(internal) RETURNS void LANGUAGE internal IMMUTABLE AS 'rle_type_validator' WITH(OID=9923, DESCRIPTION="Type speific RLE compression validator");
 
- CREATE FUNCTION gp_zstd_constructor(internal, internal, bool) RETURNS internal LANGUAGE internal VOLATILE AS 'zstd_constructor' WITH (OID=7071, DESCRIPTION="zstd compressor and decompressor constructor");
-
- CREATE FUNCTION gp_zstd_destructor(internal) RETURNS void LANGUAGE internal VOLATILE AS 'zstd_destructor' WITH(OID=7072, DESCRIPTION="zstd compressor and decompressor destructor");
-
- CREATE FUNCTION gp_zstd_compress(internal, int4, internal, int4, internal, internal) RETURNS void LANGUAGE internal IMMUTABLE AS 'zstd_compress' WITH(OID=7073, DESCRIPTION="zstd compressor");
-
- CREATE FUNCTION gp_zstd_decompress(internal, int4, internal, int4, internal, internal) RETURNS void LANGUAGE internal IMMUTABLE AS 'zstd_decompress' WITH(OID=7074, DESCRIPTION="zstd decompressor");
-
- CREATE FUNCTION gp_zstd_validator(internal) RETURNS void LANGUAGE internal IMMUTABLE AS 'zstd_validator' WITH(OID=7075, DESCRIPTION="zstdcompression validator");
-
  CREATE FUNCTION gp_dummy_compression_constructor(internal, internal, bool) RETURNS internal LANGUAGE internal VOLATILE AS 'dummy_compression_constructor' WITH (OID=7064, DESCRIPTION="Dummy compression destructor");
 
  CREATE FUNCTION gp_dummy_compression_destructor(internal) RETURNS internal LANGUAGE internal VOLATILE AS 'dummy_compression_destructor' WITH (OID=7065, DESCRIPTION="Dummy compression destructor");

src/include/catalog/pg_proc_gp.h

@@ -631,26 +631,6 @@ DESCR("Type specific RLE decompressor");
 DATA(insert OID = 9923 ( gp_rle_type_validator  PGNSP PGUID 12 1 0 0 0 f f f f f f i 1 0 2278 "2281" _null_ _null_ _null_ _null_ rle_type_validator _null_ _null_ _null_ n a ));
 DESCR("Type speific RLE compression validator");
 
-/* gp_zstd_constructor(internal, internal, bool) => internal */
-DATA(insert OID = 7071 ( gp_zstd_constructor  PGNSP PGUID 12 1 0 0 0 f f f f f f v 3 0 2281 "2281 2281 16" _null_ _null_ _null_ _null_ zstd_constructor _null_ _null_ _null_ n a ));
-DESCR("zstd compressor and decompressor constructor");
-
-/* gp_zstd_destructor(internal) => void */
-DATA(insert OID = 7072 ( gp_zstd_destructor  PGNSP PGUID 12 1 0 0 0 f f f f f f v 1 0 2278 "2281" _null_ _null_ _null_ _null_ zstd_destructor _null_ _null_ _null_ n a ));
-DESCR("zstd compressor and decompressor destructor");
-
-/* gp_zstd_compress(internal, int4, internal, int4, internal, internal) => void */
-DATA(insert OID = 7073 ( gp_zstd_compress  PGNSP PGUID 12 1 0 0 0 f f f f f f i 6 0 2278 "2281 23 2281 23 2281 2281" _null_ _null_ _null_ _null_ zstd_compress _null_ _null_ _null_ n a ));
-DESCR("zstd compressor");
-
-/* gp_zstd_decompress(internal, int4, internal, int4, internal, internal) => void */
-DATA(insert OID = 7074 ( gp_zstd_decompress  PGNSP PGUID 12 1 0 0 0 f f f f f f i 6 0 2278 "2281 23 2281 23 2281 2281" _null_ _null_ _null_ _null_ zstd_decompress _null_ _null_ _null_ n a ));
-DESCR("zstd decompressor");
-
-/* gp_zstd_validator(internal) => void */
-DATA(insert OID = 7075 ( gp_zstd_validator  PGNSP PGUID 12 1 0 0 0 f f f f f f i 1 0 2278 "2281" _null_ _null_ _null_ _null_ zstd_validator _null_ _null_ _null_ n a ));
-DESCR("zstdcompression validator");
-
 /* gp_dummy_compression_constructor(internal, internal, bool) => internal */
 DATA(insert OID = 7064 ( gp_dummy_compression_constructor  PGNSP PGUID 12 1 0 0 0 f f f f f f v 3 0 2281 "2281 2281 16" _null_ _null_ _null_ _null_ dummy_compression_constructor _null_ _null_ _null_ n a ));
 DESCR("Dummy compression destructor");

src/include/utils/builtins.h

@@ -1347,12 +1347,6 @@ extern Datum rle_type_compress(PG_FUNCTION_ARGS);
 extern Datum rle_type_decompress(PG_FUNCTION_ARGS);
 extern Datum rle_type_validator(PG_FUNCTION_ARGS);
 
-extern Datum zstd_constructor(PG_FUNCTION_ARGS);
-extern Datum zstd_destructor(PG_FUNCTION_ARGS);
-extern Datum zstd_compress(PG_FUNCTION_ARGS);
-extern Datum zstd_decompress(PG_FUNCTION_ARGS);
-extern Datum zstd_validator(PG_FUNCTION_ARGS);
-
 extern Datum delta_constructor(PG_FUNCTION_ARGS);
 extern Datum delta_destructor(PG_FUNCTION_ARGS);
 extern Datum delta_compress(PG_FUNCTION_ARGS);