提交 1315881c 编写于 作者: B Bradford D. Boyle

Remove Ivy and sync_tools from gpAux

In previous versions of Greenplum, Ivy was used to resolve and download
dependencies for building the GPDB server. As we shift towards relying
on the build environment to provided the necessary dependencies, we no
longer require Ivy.

Previous work had shifted the dependencies out of Ivy for supported
platforms (i.e., RHEL and Ubuntu) but the Concourse pipeline was still
invoking Ivy (an effective no-op) as part of the sync_tools task. This
commit removes the sync_tools task and the remaining Ivy configuration
from inside of gpAux.

The responsibility for downloading gporca was moved into the
compile_gpdb Concourse task which is where xerces-c and orca are built.
Co-authored-by: NBradford D. Boyle <bboyle@pivotal.io>
Co-authored-by: NGoutam Tadi <gtadi@pivotal.io>
上级 8e7cb232
......@@ -829,14 +829,6 @@ jobs:
resource: libsigar-centos6
- get: python-tarball
resource: python-centos6
- task: sync_tools
file: gpdb_src/concourse/tasks/sync_tools.yml
image: gpdb6-centos6-build
params:
TARGET_OS: centos
TARGET_OS_VERSION: 6
TASK_OS: centos
TASK_OS_VERSION: 6
- task: compile_gpdb
file: gpdb_src/concourse/tasks/compile_gpdb.yml
image: gpdb6-centos6-build
......@@ -871,14 +863,6 @@ jobs:
resource: libsigar-centos7
- get: python-tarball
resource: python-centos7
- task: sync_tools
file: gpdb_src/concourse/tasks/sync_tools.yml
image: gpdb6-centos7-build
params:
TARGET_OS: centos
TARGET_OS_VERSION: 7
TASK_OS: centos
TASK_OS_VERSION: 7
- task: compile_gpdb
image: gpdb6-centos7-build
file: gpdb_src/concourse/tasks/compile_gpdb.yml
......@@ -933,14 +917,6 @@ jobs:
resource: libsigar-centos6
- get: python-tarball
resource: python-centos6
- task: sync_tools
file: gpdb_src/concourse/tasks/sync_tools.yml
image: gpdb6-centos6-build
params:
TARGET_OS: centos
TARGET_OS_VERSION: 6
TASK_OS: centos
TASK_OS_VERSION: 6
- task: compile_gpdb
file: gpdb_src/concourse/tasks/compile_gpdb.yml
image: gpdb6-centos6-build
......@@ -968,14 +944,6 @@ jobs:
resource: libquicklz-ubuntu18.04
- get: python-tarball
resource: python-ubuntu18.04
- task: sync_tools
file: gpdb_src/concourse/tasks/sync_tools.yml
image: gpdb6-ubuntu18.04-build
params:
TARGET_OS: ubuntu
TARGET_OS_VERSION: 18.04
TASK_OS: ubuntu
TASK_OS_VERSION: 18.04
- task: compile_gpdb
image: gpdb6-ubuntu18.04-build
file: gpdb_src/concourse/tasks/compile_gpdb.yml
......
......@@ -191,6 +191,14 @@ function export_gpdb_clients() {
popd
}
function fetch_orca_src {
local orca_tag="${1}"
mkdir orca_src
wget --quiet --output-document=- "https://github.com/greenplum-db/gporca/archive/${orca_tag}.tar.gz" \
| tar xzf - --strip-components=1 --directory=orca_src
}
function build_xerces()
{
OUTPUT_DIR="gpdb_src/gpAux/ext/${BLD_ARCH}"
......@@ -208,12 +216,12 @@ function build_and_test_orca()
}
function _main() {
# Copy input ext dir; assuming ext doesnt exist
cp -a gpAux_ext/ext ${GPDB_SRC_PATH}/gpAux
mkdir gpdb_src/gpAux/ext
case "${TARGET_OS}" in
centos|ubuntu)
prep_env
fetch_orca_src "${ORCA_TAG}"
build_xerces
build_and_test_orca
install_deps
......
#! /bin/bash
set -eo pipefail
ROOT_DIR=`pwd`
# Get ssh private key from REMOTE_KEY, which is assumed to
# be encode in base64. We can't pass the key content directly
# since newline doesn't work well for env variable.
function setup_ssh_keys() {
# Setup ssh keys
echo -n $REMOTE_KEY | base64 -d > ~/remote.key
chmod 400 ~/remote.key
eval `ssh-agent -s`
ssh-add ~/remote.key
# Scan for target server's public key, append port number
ssh-keyscan -p $REMOTE_PORT $REMOTE_HOST > pubkey
awk '{printf "[%s]:", $1 }' pubkey > tmp
echo -n $REMOTE_PORT >> tmp
awk '{$1 = ""; print $0; }' pubkey >> tmp
mkdir ~/.ssh
cat tmp >> ~/.ssh/known_hosts
}
function remote_setup() {
# Get a session id for different commit builds.
SESSION_ID=`ssh -T -p $REMOTE_PORT $REMOTE_USER@$REMOTE_HOST 'echo \$\$'`
# Create working directory on remote machine.
ssh -T -p $REMOTE_PORT $REMOTE_USER@$REMOTE_HOST > dir <<- EOF
mkdir -p gpdb-compile/$SESSION_ID
cd gpdb-compile/$SESSION_ID
echo \$PWD
EOF
GPDB_DIR=`cat dir`
# Setup environment variables needed on remote machine.
cat >env.sh <<- EOF
export IVYREPO_HOST="$IVYREPO_HOST"
export IVYREPO_REALM="$IVYREPO_REALM"
export IVYREPO_USER="$IVYREPO_USER"
export IVYREPO_PASSWD="$IVYREPO_PASSWD"
EOF
scp -P $REMOTE_PORT -q env.sh $REMOTE_USER@$REMOTE_HOST:$GPDB_DIR/env.sh
# Get git information from local repo(concourse gpdb_src input)
cd gpdb_src
GIT_URI=`git config --get remote.origin.url`
GIT_COMMIT=`git rev-parse HEAD`
cd ..
}
# Since we're cloning in a different machine, maybe there's
# new commit pushed to the same repo. We need to reset to the
# same commit to current concourse build.
function remote_clone() {
ssh -A -T -p $REMOTE_PORT $REMOTE_USER@$REMOTE_HOST <<- EOF
cd $GPDB_DIR
git clone --recursive $GIT_URI gpdb_src
cd gpdb_src
git reset --hard $GIT_COMMIT
EOF
}
function remote_compile() {
# .profile is not automatically sourced when ssh -T to AIX
ssh -T -p $REMOTE_PORT $REMOTE_USER@$REMOTE_HOST <<- EOF
. .profile
cd $GPDB_DIR
. env.sh
cd gpdb_src/gpAux
make sync_tools
make GPROOT="$GPDB_DIR" BLD_TARGETS="$BLD_TARGETS" -s dist
mv *.zip $GPDB_DIR/
EOF
}
function download() {
scp -P $REMOTE_PORT -q $REMOTE_USER@$REMOTE_HOST:$GPDB_DIR/*.zip $ROOT_DIR/gpdb_artifacts/
}
# Since we are cloning and building on remote machine,
# files won't be deleted as concourse container destroys.
# We have to clean everything for success build.
function cleanup() {
ssh -T -p $REMOTE_PORT $REMOTE_USER@$REMOTE_HOST <<- EOF
rm -rf $GPDB_DIR
EOF
}
function _main() {
if [ -z "$REMOTE_PORT" ]; then
REMOTE_PORT=22
fi
time setup_ssh_keys
time remote_setup
time remote_clone
time remote_compile
time download
time cleanup
}
_main "$@"
#!/bin/bash -l
set -exo pipefail
CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
GPDB_SRC_PATH=${GPDB_SRC_PATH:=gpdb_src}
function make_sync_tools() {
pushd ${GPDB_SRC_PATH}/gpAux
# Requires these variables in the env:
# IVYREPO_HOST IVYREPO_REALM IVYREPO_USER IVYREPO_PASSWD
make sync_tools
popd
wget -q -O - https://github.com/greenplum-db/gporca/archive/v3.48.0.tar.gz | tar zxf - -C ${GPDB_SRC_PATH}/gpAux/ext/${BLD_ARCH}
mkdir -p orca_src
mv ${GPDB_SRC_PATH}/gpAux/ext/${BLD_ARCH}/gporca*/* orca_src/
}
function _main() {
case "${TARGET_OS}" in
centos)
case "${TARGET_OS_VERSION}" in
6)
BLD_ARCH=rhel6_x86_64
;;
7)
BLD_ARCH=rhel7_x86_64
;;
esac
;;
ubuntu)
export BLD_ARCH=ubuntu18.04_x86_64
;;
*)
echo "only centos, ubuntu are supported TARGET_OS'es"
false
;;
esac
# We have moved out of ivy for Centos{6,7} and Ubuntu never had ivy
# so make sync_tools will not create the necessary directory for centos or ubuntu.
case "${TARGET_OS}" in
centos|ubuntu)
mkdir -p ${GPDB_SRC_PATH}/gpAux/ext/${BLD_ARCH}
;;
esac
make_sync_tools
# Move ext directory to output dir.
mv ${GPDB_SRC_PATH}/gpAux/ext gpAux_ext/
}
_main "$@"
......@@ -3,14 +3,12 @@ image_resource:
type: docker-image
inputs:
- name: gpdb_src
- name: gpAux_ext
- name: libquicklz-installer
- name: libquicklz-devel-installer
optional: true
- name: libsigar-installer
optional: true
- name: python-tarball
- name: orca_src
outputs:
- name: gpdb_artifacts
run:
......@@ -21,3 +19,4 @@ params:
BLD_TARGETS:
OUTPUT_ARTIFACT_DIR: gpdb_artifacts
CONFIGURE_FLAGS:
ORCA_TAG: v3.48.0
platform: linux
image_resource:
type: docker-image
inputs:
- name: gpdb_src
outputs:
- name: gpdb_artifacts
run:
path: gpdb_src/concourse/scripts/compile_gpdb_remote.bash
params:
TARGET_OS:
TARGET_OS_VERSION:
BLD_TARGETS:
OUTPUT_ARTIFACT_DIR: gpdb_artifacts
IVYREPO_HOST:
IVYREPO_REALM:
IVYREPO_USER:
IVYREPO_PASSWD:
CONFIGURE_FLAGS:
---
platform: linux
image_resource:
type: docker-image
inputs:
- name: gpdb_src
outputs:
- name: gpAux_ext
- name: orca_src
run:
path: gpdb_src/concourse/scripts/sync_tools.bash
params:
TARGET_OS:
TARGET_OS_VERSION:
TASK_OS:
TASK_OS_VERSION:
IVYREPO_HOST:
IVYREPO_REALM:
IVYREPO_USER:
IVYREPO_PASSWD:
......@@ -40,7 +40,6 @@ all : devel
BLD_TOP:=$(shell sh -c pwd)
include Makefile.global
include Makefile.thirdparty
include releng/releng.mk
ifneq "$(BLD_BITS)" ""
export NO_M64=1
......
<project name="cdb2" default="resolve" xmlns:ivy="antlib:org.apache.ivy.ant">
<property name="ivyrepo.host" value="dummy_host"/>
<property name="ivyrepo.realm" value="dummy_realm"/>
<property name="ivyrepo.user" value="dummy_user"/>
<property name="ivyrepo.passwd" value="dummy_passwd"/>
<target name="resolve" if="BLD_ARCH">
<ivy:settings>
<credentials
host="${ivyrepo.host}" realm="${ivyrepo.realm}"
username="${ivyrepo.user}" passwd="${ivyrepo.passwd}" />
</ivy:settings>
<ivy:resolve conf="${BLD_ARCH}"
log="quiet"/>
<ivy:report conf="${BLD_ARCH}"
graph="false"/>
</target>
<target name="clean">
<ivy:cleancache />
</target>
<target name="post-resolve-trigger">
<exec executable="./ivy.sh">
<arg value="${dep.module}"/>
<arg value="${dep.revision}"/>
<arg value="${dep.organisation}"/>
<arg value="${BLD_ARCH}"/>
</exec>
</target>
<target name="download-untar-trigger">
<echo>organisation=${dep.organisation} | module=${dep.module} | revision=${dep.revision} | artifact=${dep.artifact} | origin=${dep.origin} | local=${dep.local} | size=${dep.size}</echo>
<exec executable="./ivy_util.sh">
<arg value="${dep.file}"/>
<arg value="${dep.revision}"/>
</exec>
<exec executable="./ivy.sh">
<arg value="${dep.module}"/>
<arg value="${dep.revision}"/>
<arg value="${dep.organisation}"/>
<arg value="${BLD_ARCH}"/>
</exec>
</target>
</project>
#!/bin/bash
## ======================================================================
## Post download untar trigger : For EXT
## ======================================================================
MODULE=$1
REVISION=$2
ORG=$3
BUILD_STAGE=$4
# ##
# ## Working directory is the source dir from where we do make
# ## This will be used in creating ext dir
# ##
WORKING_DIRECTORY=$(cd ${PWD}/../../..; pwd -P)
EXT_DIRECTORY=${WORKING_DIRECTORY}/ext/${BUILD_STAGE}
if [ ! -d "${EXT_DIRECTORY}" ]; then
mkdir -p ${EXT_DIRECTORY}
fi
while read line; do
if [[ $line =~ .*"${BUILD_STAGE}->"* && $line =~ ${REVISION} && $line =~ ${MODULE} && $line =~ ${ORG} ]]; then
line=`echo $line | cut -f5 -d "="`
IFS=';' read -ra dependency <<< "$line"
for i in "${dependency[@]}"; do
if [[ $i =~ .*"${BUILD_STAGE}->"* ]]; then
IFS='->' read -a array <<< "$i"
BUILD_STAGE=`echo "${array[2]}" | cut -f1 -d "\"" | cut -f2 -d ";"`
fi
done
fi
done < ivy.xml
TOOLS_DIRECTORY=/opt/releng/tools/${ORG}/${MODULE}/${REVISION}/${BUILD_STAGE}
##
## Copy all the files from /opt/tools to main/ext
##
if [ -d "${TOOLS_DIRECTORY}" ] && [ ! -f "${EXT_DIRECTORY}/checksums.${MODULE}-${REVISION}" ]; then
cd $TOOLS_DIRECTORY
tar cf - * | ( cd ${EXT_DIRECTORY}; tar xfp -)
fi
##
## Softlink all the files for ext.
##
## This does require the dirs and sub-dirs to
## to be created, which is done in the
## first for loop
##
# for dir_entry in `find . -type d`
# do
# echo "$dir_entry"
# mkdir -p ${EXT_DIRECTORY}/${BUILDSTAGE}$dir_entry
# done
# for entry in `find . -type f`
# do
# echo "$entry"
# ln -s ${TOOLS_DIRECTORY}/$entry ${EXT_DIRECTORY}/${BUILDSTAGE}$entry
# done
<ivy-module version="2.0">
<info organisation="Pivotal"
module="gpdb" />
<configurations>
<conf name="rhel6_x86_64" visibility="public"/>
<conf name="ubuntu18.04_x86_64" visibility="public"/>
<conf name="rhel7_x86_64" visibility="public"/>
<conf name="suse11_x86_64" visibility="public"/>
<conf name="sles11_x86_64" visibility="public"/>
<conf name="aix7_ppc_64" visibility="public"/>
</configurations>
<dependencies>
<dependency org="OpenSSL" name="openssl" rev="1.0.2l" conf="aix7_ppc_64->aix7_ppc_64" />
<dependency org="OpenLDAP" name="openldap" rev="2.4.45" conf="aix7_ppc_64->aix7_ppc_64" />
<dependency org="PyYAML" name="yaml" rev="0.1.7" conf="aix7_ppc_64->aix7_ppc_64" />
<dependency org="apache" name="apr" rev="1.6.2" conf="aix7_ppc_64->aix7_ppc_64" />
<dependency org="apache" name="apr-util" rev="1.6.0" conf="aix7_ppc_64->aix7_ppc_64" />
<dependency org="bzip" name="bzip2" rev="1.0.6" conf="aix7_ppc_64->aix7_ppc_64" />
<dependency org="libevent" name="libevent" rev="2.1.8" conf="aix7_ppc_64->aix7_ppc_64" />
<dependency org="mit" name="krb5" rev="1.15.1" conf="aix7_ppc_64->aix7_ppc_64" />
<dependency org="zlib" name="zlib" rev="1.2.11" conf="aix7_ppc_64->aix7_ppc_64" />
</dependencies>
</ivy-module>
#!/bin/bash
## ======================================================================
## Post download untar trigger
## ======================================================================
FILE=$1
REVISION=$2
ARTIFACT_DIR=$( dirname ${FILE} )
##
## Expand tarball
##
if [ ! -f "${FILE}" ]; then
echo "WARNING: tarball does not exist (${FILE})"
exit 2
else
pushd ${ARTIFACT_DIR}/.. > /dev/null
gunzip -qc ${FILE} | tar xf -
popd > /dev/null
if [ $? != 0 ]; then
echo "FATAL: Problem exapanding tarball (${FILE})"
exit 1
fi
fi
## ----------------------------------------------------------------------
## Devel package processing
## Restore object files into loation used for build
## ----------------------------------------------------------------------
echo "${FILE}" | grep devel
if [ $? = 0 ]; then
BASE_PULSE_PROJECT=`tar tvf ${FILE} | head -1 | sed -e 's|\(.* \./\)\(.*\)|\2|' | cut -d '/' -f1`
BASE_BUILD_STAGE=`tar tvf ${FILE} | head -1 | sed -e 's|\(.* \./\)\(.*\)|\2|' | cut -d '/' -f2`
rm -rf /opt/releng/build/${BASE_PULSE_PROJECT}/${BASE_BUILD_STAGE}
cd ..
[ ! -d /opt/releng/build/${BASE_PULSE_PROJECT} ] && mkdir -p /opt/releng/build/${BASE_PULSE_PROJECT}
ln -s ${ARTIFACT_DIR}/../${BASE_PULSE_PROJECT}/${BASE_BUILD_STAGE} /opt/releng/build/${BASE_PULSE_PROJECT}/${BASE_BUILD_STAGE}
fi
exit 0
<?xml version="1.0" encoding="UTF-8"?>
<ivysettings>
<!-- ====================================================================== -->
<caches defaultCacheDir="/opt/releng/tools"
lockStrategy="artifact-lock"
ivyPattern="[organisation]/[module]/[revision]/[type]s/ivy-[revision].xml"
artifactPattern="[organisation]/[module]/[revision]/[type]s/[artifact]-[revision].[ext]"
datafilePattern="[organisation]/[module]/ivydata-[revision].properties"
/>
<!-- ====================================================================== -->
<property name="repo.dir" value="/opt/repo"/>
<settings defaultResolver="chain-resolver"/>
<!-- ====================================================================== -->
<resolvers>
<!-- this resolver is used for integration test of third-part
library such as libhdfs3, ${HAWQ_ROOT}/repo directory should
be empty in main branch -->
<filesystem name="integration">
<ivy pattern="${ivy.settings.dir}/../../../repo/[organization]/[module]/[revision]/[type]s/ivy-[revision].xml" />
<artifact pattern="${ivy.settings.dir}/../../../repo/[organization]/[module]/[revision]/[type]s/[artifact]-[revision].[ext]" />
</filesystem>
<filesystem name="local">
<ivy pattern="${repo.dir}/[organization]/[module]/[revision]/[type]s/ivy-[revision].xml" />
<artifact pattern="${repo.dir}/[organization]/[module]/[revision]/[type]s/[artifact]-[revision].[ext]" />
</filesystem>
<url name="public">
<ivy pattern="https://repo.pivotal.io/gpdb-ext-release-local/[organization]/[module]/[revision]/[type]s/ivy-[revision].xml" />
<artifact pattern="https://repo.pivotal.io/gpdb-ext-release-local/[organization]/[module]/[revision]/[type]s/[artifact]-[revision].[ext]" />
</url>
<chain name="chain-resolver">
<resolver ref="local"/>
<resolver ref="public"/>
<resolver ref="integration"/>
</chain>
</resolvers>
<!-- ====================================================================== -->
<triggers>
<ant-call target="post-resolve-trigger"
prefix="dep"
event="post-resolve-dependency"/>
</triggers>
<triggers>
<ant-call target="download-untar-trigger"
prefix="dep"
event="post-download-artifact"
filter="type=targz AND status=successful"/>
</triggers>
<!-- ====================================================================== -->
</ivysettings>
##-------------------------------------------------------------------------------------
##
## Copyright (C) 2011 EMC - Data Computing Division (DCD)
##
## @doc: Engineering Services makefile utilities
##
## @author: eespino
##
##-------------------------------------------------------------------------------------
.PHONY: opt_write_test sync_tools clean_tools
#-------------------------------------------------------------------------------------
# machine and OS properties
#-------------------------------------------------------------------------------------
UNAME = $(shell uname)
UNAME_P = $(shell uname -p)
UNAME_M = $(shell uname -m)
UNAME_ALL = $(UNAME).$(UNAME_P)
# shared lib support
ifeq (Darwin, $(UNAME))
LDSFX = dylib
else
LDSFX = so
endif
ifeq (x86_64, $(UNAME_M))
ARCH_FLAGS = -m64
else
ARCH_FLAGS = -m32
endif
##-------------------------------------------------------------------------------------
## dependent modules
##
## NOTE: Dependent project module version is kept in $(BLD_TOP)/releng/make/dependencies/ivy.xml
##-------------------------------------------------------------------------------------
GREP_SED_VAR = $(BLD_TOP)/releng/make/dependencies/ivy.xml | sed -e 's|\(.*\)rev="\(.*\)"[ \t]*conf\(.*\)|\2|'
## ----------------------------------------------------------------------
## Sync/Clean tools
## ----------------------------------------------------------------------
## Populate/clean up dependent releng supported tools. The projects are
## downloaded and installed into /opt/releng/...
##
## Tool dependencies and platform config mappings are defined in:
## * Apache Ivy dependency definition file
## releng/make/dependencies/ivy.xml
## ----------------------------------------------------------------------
opt_write_test:
@if [ ! -e /opt/releng -o ! -w /opt/releng ] && [ ! -w /opt ]; then \
echo ""; \
echo "======================================================================"; \
echo "ERROR: /opt is not writable."; \
echo "----------------------------------------------------------------------"; \
echo " Supporting tools are stored in /opt. Please ensure you have"; \
echo " write access to /opt"; \
echo "======================================================================"; \
echo ""; \
exit 1; \
fi
# ----------------------------------------------------------------------
# Populate dependent internal and thirdparty dependencies. This
# will be retrieved and place in "ext" directory in root
# directory.
# ----------------------------------------------------------------------
sync_tools: opt_write_test
@if [ -d /opt/releng/tools ]; then \
LCK_FILES=$$( find /opt/releng/tools -name "*.lck" ); \
if [ -n "$${LCK_FILES}" ]; then \
echo "Removing existing .lck files!"; \
find /opt/releng/tools -name "*.lck" | xargs rm; \
fi \
fi
@cd releng/make/dependencies; \
(umask 002; ANT_OPTS="-Djavax.net.ssl.trustStore=$(BLD_TOP)/releng/make/dependencies/cacerts" \
/opt/releng/apache-ant/bin/ant -DBLD_ARCH=$(BLD_ARCH) \
-Divyrepo.host=$(IVYREPO_HOST) -Divyrepo.realm="$(IVYREPO_REALM)" \
-Divyrepo.user=$(IVYREPO_USER) -Divyrepo.passwd="$(IVYREPO_PASSWD)" -quiet resolve);
clean_tools: opt_write_test
@cd releng/make/dependencies; \
/opt/releng/apache-ant/bin/ant clean; \
rm -rf /opt/releng/apache-ant; \
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册