Vagrant CentOs VM with GPORCA support [#114923155]

- Change the current vagrant to clone the GPDB inside and run it, to avoid confusion between host and guest environment.
- Change the VM size to 8GB and 4 cpus to better compile and test performance
- At the end of the vagrant the GPDB will up and running with gpdemo (3 seg and 3 mirrors)
- use `vagrant up gpdb` to build GPDB with GPORCA using option --enable-orca
- use `vagrant up gpdb_without_gporca` to build GPDB without GPORCA
Signed-off-by: NXin Zhang <xzhang@pivotal.io>

vagrant/centos/Vagrantfile

-# Original Authors: Navneet Potti, Nabarun Nag and Jignesh Patel
 require 'yaml'
 
-# IP Address for the private VM network
-ip_address = "192.168.10.200"
-
 # Basic Vagrant config (API version 2)
 Vagrant.configure(2) do |config|
-
   # Base box: Centos-7 box
   # NOTE: Over time the VMI below may become outdated, so may need to be
   #       substituted with a more recent VMI
   config.vm.box = "boxcutter/centos72"
 
-  # Make this VM reachable on the host network as well, so that other
-  # VM's running other browsers can access our dev server.
-  config.vm.network :private_network, ip: ip_address
+  config.vm.hostname="gpdbvagrant"
 
   # Give a reasonable amount of cpu and memory to the VM
   config.vm.provider "virtualbox" do |vb|
     vb.name = "gpdb-dev-host" # Name in VirtualBox
-    vb.memory = 4096
-    vb.cpus = 2
+    vb.memory = 8192
+    vb.cpus = 4
+    vb.customize ["storagectl", :id, "--name", "SATA Controller", "--hostiocache", "on"]
   end
 
-  # Make the GPDB code folder will be visible as /gpdb in the virtual machine
-  config.vm.synced_folder "../../.", "/gpdb"
-
   if File.file?('vagrant-local.yml')
     local_config = YAML::load_file('vagrant-local.yml')
     local_config['synced_folder'].each do |folder|
@@ -33,10 +24,16 @@ Vagrant.configure(2) do |config|
     end
   end
 
-  # Install packages that are needed to build and run GPDB
-  config.vm.provision "shell", path: "vagrant-setup.sh"
-  config.vm.provision "shell", path: "vagrant-build.sh"
-  config.vm.provision "shell", path: "vagrant-configure-os.sh", args: ip_address
-  config.vm.provision "shell", path: "vagrant-install.sh"
+  config.vm.define("gpdb") do |gpdb|
+    gpdb.vm.provision "shell", path: "vagrant-setup.sh"
+    gpdb.vm.provision "shell", path: "vagrant-configure-os.sh"
+    gpdb.vm.provision "shell", path: "vagrant-build-gporca.sh", privileged: false
+    gpdb.vm.provision "shell", path: "vagrant-build-gpdb.sh", privileged: false, args: "--enable-orca"
+  end
 
+  config.vm.define("gpdb_without_gporca") do |gpdb_without_gporca|
+    gpdb_without_gporca.vm.provision "shell", path: "vagrant-setup.sh"
+    gpdb_without_gporca.vm.provision "shell", path: "vagrant-configure-os.sh"
+    gpdb_without_gporca.vm.provision "shell", path: "vagrant-build-gpdb.sh", privileged: false
+  end
 end

vagrant/centos/vagrant-build-gpdb.sh

+#!/bin/bash
+
+set -ex
+
+# clone the repos
+rm -fr gpdb
+
+pushd ~
+  git clone https://github.com/greenplum-db/gpdb
+popd
+
+export CC="ccache cc"
+export CXX="ccache c++"
+export PATH=/usr/local/bin:$PATH
+
+rm -rf /usr/local/gpdb
+pushd ~/gpdb
+  ./configure --prefix=/usr/local/gpdb $1 
+  make clean
+  make -j4 -s && make install
+popd
+
+# generate ssh key to avoid typing password all the time during gpdemo make
+rm -f ~/.ssh/id_rsa
+rm -f ~/.ssh/id_rsa.pub
+rm -f ~/.ssh/authorized_keys
+ssh-keygen -t rsa -N "" -f "$HOME/.ssh/id_rsa"
+cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys
+chmod 600 ~/.ssh/authorized_keys
+
+# BUG: fix the LD_LIBRARY_PATH to find installed GPOPT libraries
+echo export LD_LIBRARY_PATH=/usr/local/lib:\$LD_LIBRARY_PATH >> /usr/local/gpdb/greenplum_path.sh
+
+# use gpdemo to start the cluster
+pushd ~/gpdb/gpAux/gpdemo
+  source /usr/local/gpdb/greenplum_path.sh
+  make
+popd

vagrant/centos/vagrant-build-gporca.sh

+#!/bin/bash
+
+set -ex
+
+# clone the repos
+rm -fr gpos
+rm -fr gp-xerces
+rm -fr gporca
+
+pushd ~
+  git clone https://github.com/greenplum-db/gpos
+  git clone https://github.com/greenplum-db/gp-xerces
+  git clone https://github.com/greenplum-db/gporca
+popd
+
+export CC="ccache cc"
+export CXX="ccache c++"
+export PATH=/usr/local/bin:$PATH
+
+pushd ~/gpos
+  rm -fr build
+  mkdir build
+  pushd build
+    cmake ../
+    make -j4 && make install
+  popd
+popd
+
+pushd ~/gp-xerces
+  rm -fr build
+  mkdir build
+  pushd build
+    ../configure --prefix=/usr/local
+    make -j4 && make install
+  popd
+popd
+
+pushd ~/gporca
+  rm -fr build
+  mkdir build
+  pushd build
+    cmake ../
+    make -j4 && make install
+  popd
+popd
+

vagrant/centos/vagrant-build.sh

-#!/bin/bash
-
-sudo rm -rf /usr/local/greenplum-db
-sudo mkdir /usr/local/greenplum-db
-sudo chown -R vagrant:vagrant /usr/local/greenplum-db
-cd /gpdb
-./configure --prefix=/usr/local/greenplum-db --enable-depend --enable-debug --with-python || exit 1
-sudo make clean || exit 1
-sudo make || exit 1
-sudo make install
-sudo chown -R vagrant:vagrant /usr/local/greenplum-db
-

vagrant/centos/vagrant-configure-os.sh

@@ -2,12 +2,6 @@
 
 set -x
 
-IP_ADDRESS=$1
-if [[ -z "$IP_ADDRESS" ]]; then
-    echo "ERROR: IP address should be provided as a parameter to the script"
-    exit 1
-fi
-
 # Stop firewall
 sudo systemctl stop firewalld
 
@@ -15,15 +9,6 @@ sudo systemctl stop firewalld
 sudo setenforce 0
 sudo sed -i 's/^SELINUX=.*$/SELINUX=disabled/' /etc/selinux/config
 
-# Setting up hostname for host-private network
-sudo hostnamectl set-hostname gpdbvagrant.pivotal.io
-sudo hostname gpdbvagrant.pivotal.io
-sudo sed -i '/vagrant/d' /etc/hosts
-sudo bash -c "echo '$IP_ADDRESS gpdbvagrant.pivotal.io gpdbvagrant' >> /etc/hosts"
-sudo sed -i '/HOSTNAME/d' /etc/sysconfig/network
-sudo bash -c 'echo "HOSTNAME=gpdbvagrant.pivotal.io" >> /etc/sysconfig/network'
-sudo bash -c 'echo "DHCP_HOSTNAME=gpdbvagrant.pivotal.io" >> /etc/sysconfig/network'
-
 # GPDB Kernel Settings
 sudo rm -f /etc/sysctl.d/gpdb.conf
 sudo bash -c 'printf "# GPDB-Specific Settings\n\n"                    >> /etc/sysctl.d/gpdb.conf'
@@ -43,6 +28,7 @@ sudo bash -c 'printf "net.ipv4.tcp_tw_recycle = 1\n"                   >> /etc/s
 sudo bash -c 'printf "net.ipv4.tcp_max_syn_backlog = 4096\n"           >> /etc/sysctl.d/gpdb.conf'
 sudo bash -c 'printf "net.ipv4.conf.all.arp_filter = 1\n"              >> /etc/sysctl.d/gpdb.conf'
 sudo bash -c 'printf "net.ipv4.ip_local_port_range = 1025 65535\n"     >> /etc/sysctl.d/gpdb.conf'
+sudo bash -c 'printf "net.ipv6.conf.all.disable_ipv6 = 1\n             >> /etc/sysctl.d/gpdb.conf'
 sudo bash -c 'printf "net.core.netdev_max_backlog = 10000\n"           >> /etc/sysctl.d/gpdb.conf'
 sudo bash -c 'printf "net.core.rmem_max = 2097152\n"                   >> /etc/sysctl.d/gpdb.conf'
 sudo bash -c 'printf "net.core.wmem_max = 2097152\n"                   >> /etc/sysctl.d/gpdb.conf'

vagrant/centos/vagrant-install-inner.sh

-#!/bin/bash
-
-set -x
-
-cd ~
-rm ~/.bashrc
-printf '#!/bin/bash\n\n'                                     >> ~/.bashrc
-printf '. /etc/bashrc\n\n'                                   >> ~/.bashrc
-printf '\n# GPDB environment variables\nexport GPDB=/gpdb\n' >> ~/.bashrc
-printf 'export GPHOME=/usr/local/greenplum-db\n'             >> ~/.bashrc
-printf 'export GPDATA=/data\n'                               >> ~/.bashrc
-printf 'if [ -e $GPHOME/greenplum_path.sh ]; then\n\t'       >> ~/.bashrc
-printf 'source $GPHOME/greenplum_path.sh\nfi\n'              >> ~/.bashrc
-source ~/.bashrc
-
-killall postgres
-sleep 5
-killall postgres -s SIGKILL
-sudo rm -rf $GPDATA
-sudo mkdir -p $GPDATA
-sudo chown -R vagrant:vagrant $GPDATA
-mkdir $GPDATA/master
-mkdir $GPDATA/primary
-
-gpssh-exkeys -h `hostname`
-hostname > $GPDATA/hosts
-
-GPCFG=$GPDATA/gpinitsystem_config
-rm -f $GPCFG
-printf "declare -a DATA_DIRECTORY=($GPDATA/primary $GPDATA/primary)\n" >> $GPCFG
-printf "MASTER_HOSTNAME=`hostname`\n"                                  >> $GPCFG
-printf "MACHINE_LIST_FILE=$GPDATA/hosts\n"                             >> $GPCFG
-printf "MASTER_DIRECTORY=$GPDATA/master\n"                             >> $GPCFG
-printf "ARRAY_NAME=\"GPDB\" \n"                                        >> $GPCFG
-printf "SEG_PREFIX=gpseg\n"                                            >> $GPCFG
-printf "PORT_BASE=40000\n"                                             >> $GPCFG
-printf "MASTER_PORT=5432\n"                                            >> $GPCFG
-printf "TRUSTED_SHELL=ssh\n"                                           >> $GPCFG
-printf "CHECK_POINT_SEGMENTS=8\n"                                      >> $GPCFG
-printf "ENCODING=UNICODE\n"                                            >> $GPCFG
-
-$GPHOME/bin/gpinitsystem -a -c $GPDATA/gpinitsystem_config
-printf '# Remember the master data directory location\n' >> ~/.bashrc
-printf 'export MASTER_DATA_DIRECTORY=$GPDATA/master/gpseg-1\n' >> ~/.bashrc
-source ~/.bashrc

vagrant/centos/vagrant-install.sh

-#!/bin/bash
-
-sudo -H -u vagrant /bin/bash /vagrant/vagrant-install-inner.sh
-

vagrant/centos/vagrant-setup.sh

@@ -17,6 +17,7 @@ sudo yum -y install libyaml libyaml-devel
 sudo yum -y install epel-release
 sudo yum -y install htop
 sudo yum -y install perl-Env
+sudo yum -y install ccache
 wget https://bootstrap.pypa.io/get-pip.py
 sudo python get-pip.py
 sudo pip install psi lockfile paramiko setuptools epydoc
@@ -24,3 +25,17 @@ rm get-pip.py
 
 # Misc
 sudo yum -y install vim mc psmisc
+
+# cmake 3.0
+pushd ~
+  wget http://www.cmake.org/files/v3.0/cmake-3.0.0.tar.gz
+  tar -zxvf cmake-3.0.0.tar.gz
+  pushd cmake-3.0.0
+    ./bootstrap
+    make
+    make install
+    export PATH=/usr/local/bin:$PATH
+  popd
+popd
+
+sudo chown -R vagrant:vagrant /usr/local