From b0977a59192a06c76dbcb106ffb1a94e96118f6d Mon Sep 17 00:00:00 2001 From: Hubert Zhang Date: Mon, 11 Jun 2018 14:35:58 +0800 Subject: [PATCH] Support plpython3u in GPDB. Follow src/pl/plpython/README.md to see how to build and use plpython3u on GPDB. Co-authored-by: Yandong Yao --- README.md | 6 +++++ config/python.m4 | 11 +++++---- configure | 10 +++++--- gpMgmt/bin/Makefile | 2 +- src/pl/plpython/Makefile | 2 +- src/pl/plpython/README.md | 48 ++++++++++++++++++++++++++++++++++++++ src/pl/plpython/plpython.c | 5 ++++ 7 files changed, 75 insertions(+), 9 deletions(-) create mode 100644 src/pl/plpython/README.md diff --git a/README.md b/README.md index 3377fea4f9..4452aef7a7 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,12 @@ See [more information about gpperfmon here](gpAux/gpperfmon/README.md) gpperfmon is dependent on several libraries like apr, apu, and libsigar +### Building GPDB with Python3 enabled + +GPDB supports Python3 with plpython3u UDF + +See [how to enable Python3](src/pl/plpython/README.md) for details. + ## Development with Docker See [README.docker.md](README.docker.md). diff --git a/config/python.m4 b/config/python.m4 index ad6b5cb09a..6668ec2175 100644 --- a/config/python.m4 +++ b/config/python.m4 @@ -32,7 +32,12 @@ fi AC_MSG_CHECKING([Python configuration directory]) python_majorversion=`${PYTHON} -c "import sys; print(sys.version[[0]])"` python_version=`${PYTHON} -c "import sys; print(sys.version[[:3]])"` -python_configdir=`${PYTHON} -c "from distutils.sysconfig import get_python_lib as f; import os; print(os.path.join(f(plat_specific=1,standard_lib=1),'config'))"` +if [ "${python_majorversion}" = "2" ]; +then + python_configdir=`${PYTHON} -c "from distutils.sysconfig import get_python_lib as f; import os; print(os.path.join(f(plat_specific=1,standard_lib=1),'config'))"` +else + python_configdir=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBPL'))))"` +fi python_includespec=`${PYTHON} -c "import distutils.sysconfig; print('-I'+distutils.sysconfig.get_python_inc())"` AC_SUBST(python_majorversion)[]dnl @@ -58,9 +63,7 @@ AC_MSG_CHECKING([how to link an embedded Python application]) python_libdir=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBDIR'))))"` python_ldlibrary=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LDLIBRARY'))))"` -python_so=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('SO'))))"` -ldlibrary=`echo "${python_ldlibrary}" | sed "s/${python_so}$//"` - +ldlibrary=`echo "${python_ldlibrary}" | sed -e 's/\.so$//' -e 's/\.dll$//' -e 's/\.dylib$//' -e 's/\.sl$//'` if test x"${python_libdir}" != x"" -a x"${python_ldlibrary}" != x"" -a x"${python_ldlibrary}" != x"${ldlibrary}" then # New way: use the official shared library diff --git a/configure b/configure index 7c45c59155..f04a0613c1 100755 --- a/configure +++ b/configure @@ -9423,7 +9423,12 @@ fi $as_echo_n "checking Python configuration directory... " >&6; } python_majorversion=`${PYTHON} -c "import sys; print(sys.version[0])"` python_version=`${PYTHON} -c "import sys; print(sys.version[:3])"` -python_configdir=`${PYTHON} -c "from distutils.sysconfig import get_python_lib as f; import os; print(os.path.join(f(plat_specific=1,standard_lib=1),'config'))"` +if [ "${python_majorversion}" = "2" ]; +then + python_configdir=`${PYTHON} -c "from distutils.sysconfig import get_python_lib as f; import os; print(os.path.join(f(plat_specific=1,standard_lib=1),'config'))"` +else + python_configdir=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBPL'))))"` +fi python_includespec=`${PYTHON} -c "import distutils.sysconfig; print('-I'+distutils.sysconfig.get_python_inc())"` # This should be enough of a message. @@ -9436,8 +9441,7 @@ $as_echo_n "checking how to link an embedded Python application... " >&6; } python_libdir=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBDIR'))))"` python_ldlibrary=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LDLIBRARY'))))"` -python_so=`${PYTHON} -c "import distutils.sysconfig,string; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('SO'))))"` -ldlibrary=`echo "${python_ldlibrary}" | sed "s/${python_so}$//"` +ldlibrary=`echo "${python_ldlibrary}" | sed -e 's/\.so$//' -e 's/\.dll$//' -e 's/\.dylib$//' -e 's/\.sl$//'` if test x"${python_libdir}" != x"" -a x"${python_ldlibrary}" != x"" -a x"${python_ldlibrary}" != x"${ldlibrary}" then diff --git a/gpMgmt/bin/Makefile b/gpMgmt/bin/Makefile index b3d0d35053..deab059296 100644 --- a/gpMgmt/bin/Makefile +++ b/gpMgmt/bin/Makefile @@ -187,7 +187,7 @@ SETUP_TOOLS_DIR=setuptools-$(SETUP_TOOLS_VERSION) PARSE_DIR=parse-$(PARSE_VERSION) ARG_PARSE_DIR=argparse-$(ARG_PARSE_VERSION) PYTHONSRC_INSTALL=$(PYLIB_SRC_EXT)/install -PYTHON_VERSION=$(shell python -c "import sys; print '%s.%s' % (sys.version_info[0:2])") +PYTHON_VERSION=$(shell python -c "import sys; print ('%s.%s' % (sys.version_info[0:2]))") PYTHONSRC_INSTALL_SITE=$(PYLIB_SRC_EXT)/install/lib/python$(PYTHON_VERSION)/site-packages PYTHONSRC_INSTALL_PYTHON_PATH=$(PYTHONPATH):$(PYTHONSRC_INSTALL_SITE) BEHAVE_BIN=$(PYTHONSRC_INSTALL)/bin/behave diff --git a/src/pl/plpython/Makefile b/src/pl/plpython/Makefile index 1b62ec828c..e6e70be285 100644 --- a/src/pl/plpython/Makefile +++ b/src/pl/plpython/Makefile @@ -29,7 +29,7 @@ shared_libpython = yes #override python_additional_libs = # sys.prefix reflects the runtime location of the python tree; # the GP install has the dylib in the lib directory of that tree. -override python_libspec = -L$(shell ${PYTHON} -c "import sys, os; print os.path.join(sys.prefix, 'lib')") -lpython${python_version} +override python_libspec = -L$(shell ${PYTHON} -c "import sys, os; print (os.path.join(sys.prefix, 'lib'))") -lpython${python_version} endif override python_libspec := -L${python_configdir}/../.. ${python_libspec} diff --git a/src/pl/plpython/README.md b/src/pl/plpython/README.md new file mode 100644 index 0000000000..14d746e021 --- /dev/null +++ b/src/pl/plpython/README.md @@ -0,0 +1,48 @@ +# Python3 Support +## How to build: +``` +# Install Python3 (take centos7 as example) +yum install -y python36-devel + +# Configure build environment to install at /usr/local/gpdb +PYTHON=/usr/bin/python3.6 ./configure --with-perl --with-python --with-libxml --prefix=/usr/local/gpdb + +# Compile and install +make -j8 +make -j8 install +``` + + +## How to use: +``` +# Ensure your environment include the SAME python version as the build environment, in our example is Python3.6. +# If you install Python in a non-system folder, you also need to add $PYTHONHOME/lib into your shared library path. +yum install -y python36-devel + +# Install Python3 packages: +wget https://bootstrap.pypa.io/get-pip.py +python3 get-pip.py +pip3 install numpy + +# Test Python3 support in GPDB +CREATE LANGUAGE plpython3u; +CREATE FUNCTION import_succeed() returns text AS $$ + import sys + import numpy + return "succeeded, as expected" +$$ LANGUAGE plpython3u; +SELECT import_succeed(); + +CREATE TYPE named_value AS ( + name text, + value integer); + +CREATE OR REPLACE FUNCTION make_pair_sets (name text) +RETURNS SETOF named_value AS $$ + import numpy as np + return ((name, i) for i in np.arange(1)) +$$ LANGUAGE plpython3u; + +SELECT * from make_pair_sets('test'); +``` + diff --git a/src/pl/plpython/plpython.c b/src/pl/plpython/plpython.c index ae47dc966b..60216c0e0b 100644 --- a/src/pl/plpython/plpython.c +++ b/src/pl/plpython/plpython.c @@ -4605,6 +4605,11 @@ _PG_init(void) #if PY_MAJOR_VERSION >= 3 PyImport_AppendInittab("plpy", PyInit_plpy); + /* PYTHONPATH and PYTHONHOME has been set to GPDB's python2.7 in Postmaster when + * gpstart. So for plpython3u, we need to unset PYTHONPATH and PYTHONHOME. + */ + unsetenv("PYTHONPATH"); + unsetenv("PYTHONHOME"); #endif Py_Initialize(); #if PY_MAJOR_VERSION >= 3 -- GitLab