Support plpython3u in GPDB.

Follow src/pl/plpython/README.md to see how to build
 and use plpython3u on GPDB.
Co-authored-by: NYandong Yao <yyao@pivotal.io>

README.md

@@ -185,6 +185,12 @@ See [more information about gpperfmon here](gpAux/gpperfmon/README.md)
 
 gpperfmon is dependent on several libraries like apr, apu, and libsigar
 
+### Building GPDB with Python3 enabled
+
+GPDB supports Python3 with plpython3u UDF
+
+See [how to enable Python3](src/pl/plpython/README.md) for details.
+
 ## Development with Docker
 
 See [README.docker.md](README.docker.md).

config/python.m4

@@ -32,7 +32,12 @@ fi
 AC_MSG_CHECKING([Python configuration directory])
 python_majorversion=`${PYTHON} -c "import sys; print(sys.version[[0]])"`
 python_version=`${PYTHON} -c "import sys; print(sys.version[[:3]])"`
-python_configdir=`${PYTHON} -c "from distutils.sysconfig import get_python_lib as f; import os; print(os.path.join(f(plat_specific=1,standard_lib=1),'config'))"`
+if [ "${python_majorversion}" = "2" ];
+then
+	python_configdir=`${PYTHON} -c "from distutils.sysconfig import get_python_lib as f; import os; print(os.path.join(f(plat_specific=1,standard_lib=1),'config'))"`
+else
+	python_configdir=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBPL'))))"`
+fi
 python_includespec=`${PYTHON} -c "import distutils.sysconfig; print('-I'+distutils.sysconfig.get_python_inc())"`
 
 AC_SUBST(python_majorversion)[]dnl
@@ -58,9 +63,7 @@ AC_MSG_CHECKING([how to link an embedded Python application])
 
 python_libdir=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBDIR'))))"`
 python_ldlibrary=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LDLIBRARY'))))"`
-python_so=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('SO'))))"`
-ldlibrary=`echo "${python_ldlibrary}" | sed "s/${python_so}$//"`
-
+ldlibrary=`echo "${python_ldlibrary}" | sed -e 's/\.so$//' -e 's/\.dll$//' -e 's/\.dylib$//' -e 's/\.sl$//'`
 if test x"${python_libdir}" != x"" -a x"${python_ldlibrary}" != x"" -a x"${python_ldlibrary}" != x"${ldlibrary}"
 then
 	# New way: use the official shared library

configure

@@ -9423,7 +9423,12 @@ fi
 $as_echo_n "checking Python configuration directory... " >&6; }
 python_majorversion=`${PYTHON} -c "import sys; print(sys.version[0])"`
 python_version=`${PYTHON} -c "import sys; print(sys.version[:3])"`
-python_configdir=`${PYTHON} -c "from distutils.sysconfig import get_python_lib as f; import os; print(os.path.join(f(plat_specific=1,standard_lib=1),'config'))"`
+if [ "${python_majorversion}" = "2" ];
+then
+	python_configdir=`${PYTHON} -c "from distutils.sysconfig import get_python_lib as f; import os; print(os.path.join(f(plat_specific=1,standard_lib=1),'config'))"`
+else
+	python_configdir=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBPL'))))"`
+fi
 python_includespec=`${PYTHON} -c "import distutils.sysconfig; print('-I'+distutils.sysconfig.get_python_inc())"`
 
 # This should be enough of a message.
@@ -9436,8 +9441,7 @@ $as_echo_n "checking how to link an embedded Python application... " >&6; }
 
 python_libdir=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBDIR'))))"`
 python_ldlibrary=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LDLIBRARY'))))"`
-python_so=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('SO'))))"`
-ldlibrary=`echo "${python_ldlibrary}" | sed "s/${python_so}$//"`
+ldlibrary=`echo "${python_ldlibrary}" | sed -e 's/\.so$//' -e 's/\.dll$//' -e 's/\.dylib$//' -e 's/\.sl$//'`
 
 if test x"${python_libdir}" != x"" -a x"${python_ldlibrary}" != x"" -a x"${python_ldlibrary}" != x"${ldlibrary}"
 then

gpMgmt/bin/Makefile

@@ -187,7 +187,7 @@ SETUP_TOOLS_DIR=setuptools-$(SETUP_TOOLS_VERSION)
 PARSE_DIR=parse-$(PARSE_VERSION)
 ARG_PARSE_DIR=argparse-$(ARG_PARSE_VERSION)
 PYTHONSRC_INSTALL=$(PYLIB_SRC_EXT)/install
-PYTHON_VERSION=$(shell python -c "import sys; print '%s.%s' % (sys.version_info[0:2])")
+PYTHON_VERSION=$(shell python -c "import sys; print ('%s.%s' % (sys.version_info[0:2]))")
 PYTHONSRC_INSTALL_SITE=$(PYLIB_SRC_EXT)/install/lib/python$(PYTHON_VERSION)/site-packages
 PYTHONSRC_INSTALL_PYTHON_PATH=$(PYTHONPATH):$(PYTHONSRC_INSTALL_SITE)
 BEHAVE_BIN=$(PYTHONSRC_INSTALL)/bin/behave

src/pl/plpython/Makefile

@@ -29,7 +29,7 @@ shared_libpython = yes
 #override python_additional_libs =
 # sys.prefix reflects the runtime location of the python tree;
 # the GP install has the dylib in the lib directory of that tree.
-override python_libspec = -L$(shell ${PYTHON} -c "import sys, os; print os.path.join(sys.prefix, 'lib')") -lpython${python_version}
+override python_libspec = -L$(shell ${PYTHON} -c "import sys, os; print (os.path.join(sys.prefix, 'lib'))") -lpython${python_version}
 endif
 
 override python_libspec := -L${python_configdir}/../.. ${python_libspec}

src/pl/plpython/README.md

+# Python3 Support
+## How to build: 
+```
+# Install Python3 (take centos7 as example)
+yum install -y python36-devel
+
+# Configure build environment to install at /usr/local/gpdb
+PYTHON=/usr/bin/python3.6 ./configure --with-perl --with-python --with-libxml --prefix=/usr/local/gpdb
+
+# Compile and install
+make -j8
+make -j8 install
+```
+
+
+## How to use:
+```
+# Ensure your environment include the SAME python version as the build environment, in our example is Python3.6.
+# If you install Python in a non-system folder, you also need to add $PYTHONHOME/lib into your shared library path.
+yum install -y python36-devel
+
+# Install Python3 packages:
+wget https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py
+pip3 install numpy
+
+# Test Python3 support in GPDB 
+CREATE LANGUAGE plpython3u;
+CREATE FUNCTION import_succeed() returns text AS $$
+  import sys
+  import numpy
+  return "succeeded, as expected"
+$$ LANGUAGE plpython3u;
+SELECT import_succeed();
+
+CREATE TYPE named_value AS (
+         name  text,
+         value  integer);
+
+CREATE OR REPLACE FUNCTION make_pair_sets (name text)
+RETURNS SETOF named_value AS $$
+  import numpy as np
+  return ((name, i) for i in np.arange(1))
+$$ LANGUAGE plpython3u;
+
+SELECT * from make_pair_sets('test');
+```
+

src/pl/plpython/plpython.c

@@ -4605,6 +4605,11 @@ _PG_init(void)
 
 #if PY_MAJOR_VERSION >= 3
 	PyImport_AppendInittab("plpy", PyInit_plpy);
+	/* PYTHONPATH and PYTHONHOME has been set to GPDB's python2.7 in Postmaster when
+	 * gpstart. So for plpython3u, we need to unset PYTHONPATH and PYTHONHOME.
+	 */
+	unsetenv("PYTHONPATH");
+	unsetenv("PYTHONHOME");
 #endif
 	Py_Initialize();
 #if PY_MAJOR_VERSION >= 3