Enable zstd by default

- set config flags to disable zstd in platforms that don't have the
  library baked in
  - currently only centos6 and centos7 have zstd provided in the build
    images
- delete the suse10 config flags, because it is no longer a supported
  platform
- add EXTRA_CONFIG_FLAGS to deb_create_package.bash
Co-authored-by: NTaylor Vesely <tvesely@pivotal.io>
Co-authored-by: NBen Christel <bchristel@pivotal.io>
Co-authored-by: NKaren Huddleston <khuddleston@pivotal.io>

.travis.yml

@@ -70,7 +70,19 @@ env:
 
 script:
   - cd ${TRAVIS_BUILD_DIR}
-  - ./configure --with-openssl --with-ldap --with-libcurl --prefix=${TRAVIS_BUILD_DIR}/gpsql --disable-orca --disable-gpcloud --enable-pxf --enable-mapreduce --with-perl --enable-orafce
+  - |
+    ./configure \
+        --with-openssl \
+        --with-ldap \
+        --with-libcurl \
+        --prefix=${TRAVIS_BUILD_DIR}/gpsql \
+        --disable-orca \
+        --disable-gpcloud \
+        --enable-pxf \
+        --enable-mapreduce \
+        --with-perl \
+        --enable-orafce \
+        --without-zstd
 
   - make
   - make install

concourse/pipelines/templates/gpdb-tpl.yml

@@ -1721,6 +1721,8 @@ jobs:
     image: centos6-test-gpdb6
     params:
       TEST_OS: sles12
+      # TODO: remove this line as soon as zstd is vendored in the binary installer for SLES12
+      CONFIGURE_FLAGS: --without-zstd
     on_success:
       <<: *ccp_destroy
   ensure:

concourse/scripts/builds/GpBuild.py

@@ -15,6 +15,8 @@ class GpBuild(GpdbBuildBase):
                                     "--with-perl",
                                     "--with-libxml",
                                     "--with-python",
+                                    # TODO: Remove this line as soon as zstd is built into Ubuntu docker image
+                                    "--without-zstd",
                                     "--prefix={0}".format(INSTALL_DIR)
                                   ]
         self.source_gcc_env_cmd = ''

concourse/scripts/common.bash

@@ -26,6 +26,8 @@ function configure() {
       # on these options for deciding what to test. Since we don't ship
       # Perl on SLES we must also skip GPMapreduce as it uses pl/perl.
       if [ "$TEST_OS" == "sles" ]; then
+        # TODO: remove this line as soon as the SLES image has zstd baked in
+        CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --without-zstd"
         ./configure --prefix=/usr/local/greenplum-db-devel --with-python --with-libxml --enable-orafce --disable-orca ${CONFIGURE_FLAGS}
       else
         ./configure --prefix=/usr/local/greenplum-db-devel --with-perl --with-python --with-libxml --enable-mapreduce --enable-orafce --disable-orca ${CONFIGURE_FLAGS}

concourse/scripts/compile_gpdb_open_source_centos.bash

@@ -46,7 +46,7 @@ function build_gpdb() {
   pushd gpdb_src
     source /opt/gcc_env.sh
     CC=$(which gcc) CXX=$(which g++) ./configure --enable-mapreduce --with-perl --with-libxml \
-	--disable-orca --with-python --disable-gpfdist --with-zstd --prefix=${GREENPLUM_INSTALL_DIR}
+        --disable-orca --with-python --disable-gpfdist --with-zstd --prefix=${GREENPLUM_INSTALL_DIR}
     # Use -j4 to speed up the build. (Doesn't seem worth trying to guess a better
     # value based on number of CPUs or anything like that. Going above -j4 wouldn't
     # make it much faster, and -j4 is small enough to not hurt too badly even on

concourse/scripts/compile_gpdb_open_source_ubuntu.bash

@@ -28,6 +28,8 @@ function install_external_depends() {
 function build_gpdb() {
     build_external_depends
     pushd gpdb_src
+        # TODO: remove this flag after adding zstd to the ubuntu build images
+        CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --without-zstd"
         CWD=$(pwd)
         LD_LIBRARY_PATH=${CWD}/depends/build/lib CC=$(which gcc) CXX=$(which g++) ./configure --enable-mapreduce --enable-orafce --with-gssapi --with-perl --with-libxml \
           --with-python \

concourse/scripts/deb_create_package.bash

@@ -26,6 +26,11 @@ report_error() {
 
 pushd ${SRC_DIR}
     VERSION=`./getversion | tr " " "."`-oss
+
+    # EXTRA_CONFIGURE_FLAGS is used by debuild; see the debian-release repo
+    # TODO: remove the EXTRA_CONFIGURE_FLAGS once zstd is added to the ubuntu docker images
+    export EXTRA_CONFIGURE_FLAGS="--without-zstd"
+
     SHA=`git rev-parse --short HEAD`
     MESSAGE="Bumping to Greenplum version $VERSION, git SHA $SHA"
     PACKAGE=`cat debian/control | egrep "^Package: " | cut -d " " -f 2`

concourse/scripts/ic_gpdb_sles12.bash

@@ -39,10 +39,12 @@ setup_demo_cluster() {
         cp -a ${SRCDIR_OLD} ${SRCDIR_NEW}
 
         cd ${SRCDIR_NEW}
+        # TODO: remove `--without-zstd` as soon as zstd is vendored in the binary installer for SLES12
         ./configure --prefix=/usr/local/greenplum-db-devel \
             --with-python --disable-orca --without-readline \
             --without-zlib --disable-gpfdist --without-libcurl \
-            --disable-pxf --without-libbz2 --enable-orafce
+            --disable-pxf --without-libbz2 --enable-orafce \
+            --without-zstd
 
         export DEFAULT_QD_MAX_CONNECT=150
         export STATEMENT_MEM=250MB

concourse/scripts/ic_gpdb_ubuntu.bash

@@ -12,6 +12,8 @@ function load_transfered_bits_into_install_dir() {
 
 function configure() {
   pushd gpdb_src
+    # TODO: remove this line as soon as zstd is vendored in the installer for ubuntu
+    CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --without-zstd"
     ./configure --prefix=${GREENPLUM_INSTALL_DIR} --with-gssapi --with-perl --with-python --with-libxml --enable-mapreduce --enable-orafce --disable-orca ${CONFIGURE_FLAGS}
   popd
 }

concourse/scripts/test_gpdb.py

@@ -53,6 +53,8 @@ def configure():
                             "--with-perl",
                             "--with-libxml",
                             "--with-python",
+                            # TODO: remove this flag after zstd is vendored in the installer for ubuntu
+                            "--without-zstd",
                             "--with-libs=/usr/local/gpdb/lib",
                             "--with-includes=/usr/local/gpdb/include",
                             "--prefix=/usr/local/gpdb"], env=p_env, cwd="gpdb_src")

configure

@@ -1593,7 +1593,7 @@ Optional Packages:
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
   --without-libbz2        do not use bzip2
-  --with-zstd             build with Zstandard support (requires zstd library)
+  --without-zstd          do not build with Zstandard
   --without-rt            do not use Realtime Library
   --without-libcurl       do not use libcurl
   --with-apr-config=PATH  path to apr-1-config utility
@@ -8133,7 +8133,7 @@ if test "${with_zstd+set}" = set; then :
   esac
 
 else
-  with_zstd=no
+  with_zstd=yes
 
 fi
 

configure.in

@@ -1099,8 +1099,8 @@ AC_SUBST(with_libbz2)
 #
 # zstd
 #
-PGAC_ARG_BOOL(with, zstd, no,
-              [build with Zstandard support (requires zstd library)])
+PGAC_ARG_BOOL(with, zstd, yes,
+              [do not build with Zstandard])
 AC_SUBST(with_zstd)
 
 #

gpAux/Makefile

@@ -159,7 +159,7 @@ endif
 
 APU_CONFIG=--with-apu-config=$(BLD_THIRDPARTY_BIN_DIR)/apu-1-config
 
-aix7_ppc_64_CONFIGFLAGS=--disable-gpcloud --without-readline --without-libcurl --disable-orca --disable-pxf $(APR_CONFIG)
+aix7_ppc_64_CONFIGFLAGS=--disable-gpcloud --without-readline --without-libcurl --disable-orca --disable-pxf --without-zstd $(APR_CONFIG)
 win32_CONFIGFLAGS=--with-gssapi --without-libcurl --disable-orca --disable-pxf --disable-gpcloud --without-libbz2 $(APR_CONFIG)
 sol10_x86_64_CONFIGFLAGS= --with-libxml $(APR_CONFIG)
 rhel6_x86_64_CONFIGFLAGS=--enable-tap-tests --enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce ${ORCA_CONFIG} --with-libxml $(APR_CONFIG) $(APU_CONFIG)
@@ -167,12 +167,11 @@ rhel7_x86_64_CONFIGFLAGS=--enable-tap-tests --enable-gpperfmon --with-gssapi --e
 #suse10_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi ${ORCA_CONFIG} --with-libxml $(APR_CONFIG) $(APU_CONFIG)
 #suse11_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi ${ORCA_CONFIG} --with-libxml $(APR_CONFIG) $(APU_CONFIG)
 #sles11_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi ${ORCA_CONFIG} --with-libxml $(APR_CONFIG) $(APU_CONFIG)
-suse10_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi ${ORCA_CONFIG} --with-libxml $(APR_CONFIG) $(APU_CONFIG)
-suse11_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi ${ORCA_CONFIG} --with-libxml --enable-orafce $(APR_CONFIG) $(APU_CONFIG)
-sles11_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi ${ORCA_CONFIG} --with-libxml --enable-orafce $(APR_CONFIG) $(APU_CONFIG)
+suse11_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi ${ORCA_CONFIG} --with-libxml --enable-orafce --without-zstd $(APR_CONFIG) $(APU_CONFIG)
+sles11_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi ${ORCA_CONFIG} --with-libxml --enable-orafce --without-zstd $(APR_CONFIG) $(APU_CONFIG)
 linux_x86_64_CONFIGFLAGS=${ORCA_CONFIG} --with-libxml $(APR_CONFIG)
 osx106_x86_CONFIGFLAGS=${ORCA_CONFIG} --with-libxml $(APR_CONFIG)
-ubuntu1604_amd64_CONFIGFLAGS=--enable-tap-tests --enable-gpperfmon --with-gssapi --enable-mapreduce ${ORCA_CONFIG} --with-libxml $(APR_CONFIG) $(APU_CONFIG)
+ubuntu1604_amd64_CONFIGFLAGS=--enable-tap-tests --enable-gpperfmon --with-gssapi --enable-mapreduce --without-zstd ${ORCA_CONFIG} --with-libxml $(APR_CONFIG) $(APU_CONFIG)
 
 BLD_CONFIGFLAGS=$($(BLD_ARCH)_CONFIGFLAGS)
 

src/backend/access/common/reloptions_gp.c

@@ -1221,7 +1221,7 @@ validateAppendOnlyRelOptions(bool ao,
 			ereport(ERROR,
 					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 					 errmsg("Zstandard library is not supported by this build"),
-					 errhint("Compile with --with-zstd to use Zstandard compression.")));
+					 errhint("Compile without --without-zstd to use Zstandard compression.")));
 #endif
 			if (complevel < 0 || complevel > 19)
 				ereport(ERROR,

src/include/pg_config.h.in

@@ -106,9 +106,6 @@
 /* Define to 1 if you have the <atomic.h> header file. */
 #undef HAVE_ATOMIC_H
 
-/* Define to 1 if you have the `ASN1_STRING_get0_data' function. */
-#undef HAVE_ASN1_STRING_GET0_DATA
-
 /* Define to 1 if you have the `BIO_meth_new' function. */
 #undef HAVE_BIO_METH_NEW
 
@@ -409,10 +406,6 @@
 /* Define to 1 if you have the `zstd' library (-lzstd). */
 #undef HAVE_LIBZSTD
 
-/* Define to 1 if constants of type 'long long int' should have the suffix LL.
-   */
-#undef HAVE_LL_CONSTANTS
-
 /* Define to 1 if the system has the type `locale_t'. */
 #undef HAVE_LOCALE_T
 
@@ -440,12 +433,12 @@
 /* Define to 1 if the system has the type `MINIDUMP_TYPE'. */
 #undef HAVE_MINIDUMP_TYPE
 
-/* Define to 1 if you have the <naucrates/init.h> header file. */
-#undef HAVE_NAUCRATES_INIT_H
-
 /* Define to 1 if you have the `mkdtemp' function. */
 #undef HAVE_MKDTEMP
 
+/* Define to 1 if you have the <naucrates/init.h> header file. */
+#undef HAVE_NAUCRATES_INIT_H
+
 /* Define to 1 if you have the <netinet/in.h> header file. */
 #undef HAVE_NETINET_IN_H