Remove more unused concourse scripts.

Thanks to Corbin Halliwill for identifying these.

concourse/build_without_orca.yml

-platform: linux
-image: docker:///yolo/gpdbdev#centos6
-inputs:
-  - name: gpdb_src
-run:
-  path: gpdb_src/concourse/scripts/build_gpdb.py
-  args:
-    - --output_dir=gpdb_src/install

concourse/perf_pipeline.yml

-#############
-# RESOURCES #
-#############
-resources:
-- name: gpdb_src
-  type: git
-  source:
-    branch: master
-    uri: https://github.com/greenplum-db/gpdb
-
-- name: instance_ids
-  type: s3
-  source:
-    access_key_id: {{aws-access-key-id}}
-    bucket: {{bucket-name}}
-    region_name: {{aws-region}}
-    secret_access_key: {{aws-secret-access-key}}
-    versioned_file: instance_ids.txt
-
-- name: installer_rhel5_gpdb_rc
-  type: s3
-  source:
-    access_key_id: {{aws_toolsmiths_access_key_id}}
-    bucket: gpdb-4.3-stable-concourse
-    private: true
-    region_name: us-west-2
-    secret_access_key: {{aws_toolsmiths_secret_access_key}}
-    versioned_file: deliverables/greenplum-db-4.3.9.1-build-1-rhel5-x86_64.zip
-
-########
-# JOBS #
-########
-jobs:
-- name: provision
-  max_in_flight: 1
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      params:
-        submodules: none
-      trigger: false  # At this stage we don't want create/delete VMs for each commit
-  - task: provision_instance
-    file: gpdb_src/concourse/provision.yml
-    params: &aws_config
-      AWS_ACCESS_KEY : {{aws-access-key-id}}
-      AWS_SECRET_KEY : {{aws-secret-access-key}}
-      AWS_KEYPAIR : "qp_perf"
-      AWS_KEYPAIR_VALUE : {{aws_perf_key}}
-      AWS_REGION : {{aws-region}}
-      AMI : "ami-7f7ab01f"
-      INSTANCE_TYPE : "t2.large"
-      INSTANCE_NAME : "qp-perf-test0"
-      SUBNET_ID : "subnet-370a1940"
-      SECURITY_GROUP_ID : "sg-d074ceb7"
-      SSH_USER : "centos"
-      TENANCY : "default"
-      WAIT_SECS : "30"
-      RETRIES : "10"
-    on_failure: *unprovision
-  - put: instance_ids
-    params:
-      file: instance_ids/instance_ids.txt
-
-- name: perf
-  max_in_flight: 1
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      passed: [provision]
-      params:
-        submodules: none
-    - get: instance_ids
-      passed: [provision]
-      trigger: true
-    - get: installer_rhel5_gpdb_rc
-  - task: setup_gpdb_remote_with_installer
-    input_mapping:
-      gpdb_installer: installer_rhel5_gpdb_rc
-    on_failure: *unprovision
-    file: gpdb_src/concourse/setup_gpdb_with_installer.yml
-    params: *aws_config
-
-- name: unprovision
-  max_in_flight: 1
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      passed: [perf]
-      params:
-        submodules: none
-    - get: instance_ids
-      passed: [perf]
-      trigger: true
-  - &unprovision
-    task: unprovision_instance
-    file: gpdb_src/concourse/unprovision.yml
-    params: *aws_config

concourse/pipeline.yml

-resources:
-  - name: gpdb_src
-    type: git
-    source:
-      uri: https://github.com/greenplum-db/gpdb
-      branch: master
-
-  - name: gpos_github_release
-    type: github-release
-    source:
-      user: greenplum-db
-      repository: gpos
-      access_token: {{gposbot_access_token}}
-
-  - name: orca_github_release
-    type: github-release
-    source:
-      user: greenplum-db
-      repository: gporca
-      access_token: {{gporcabot_access_token}}
-
-  - name: bin_xerces
-    type: s3
-    source:
-      access_key_id: {{aws-access-key-id}}
-      bucket: {{bucket-name}}
-      region_name: {{aws-region}}
-      secret_access_key: {{aws-secret-access-key}}
-      versioned_file: bin_xerces_centos5.tar.gz
-
-  - name: bin_gpdb
-    type: s3
-    source:
-      access_key_id: {{aws-access-key-id}}
-      bucket: gporca-concourse-bucket
-      region_name: {{aws-region}}
-      secret_access_key: {{aws-secret-access-key}}
-      versioned_file: bin_gpdb_with_orca_centos6.tar.gz
-
-  - name: regression_diffs
-    type: s3
-    source:
-       access_key_id: {{aws-access-key-id}}
-       bucket: {{bucket-name}}
-       region_name: {{aws-region}}
-       secret_access_key: {{aws-secret-access-key}}
-       versioned_file: regression.diffs
-
-  - name: regression_out
-    type: s3
-    source:
-       access_key_id: {{aws-access-key-id}}
-       bucket: {{bucket-name}}
-       region_name: {{aws-region}}
-       secret_access_key: {{aws-secret-access-key}}
-       versioned_file: regression.out
-
-
-jobs:
-  - name: build_with_orca
-    plan:
-      - aggregate:
-        - get: gpos_github_release
-          params:
-            globs:
-              - bin_gpos_centos5_release.tar.gz
-        - get: orca_github_release
-          params:
-            globs:
-              - bin_orca_centos5_release.tar.gz
-        - get: bin_xerces
-        - get: gpdb_src
-          params:
-            submodules: none
-          trigger: true
-      - task: build_with_orca
-        file: gpdb_src/concourse/build_with_orca.yml
-        input_mapping:
-          bin_orca: orca_github_release
-          bin_gpos: gpos_github_release
-      - task: package_tarball
-        file: gpdb_src/concourse/package_tarball.yml
-      - put: bin_gpdb
-        params:
-          file: package_tarball/bin_gpdb.tar.gz
-
-  - name: icg
-    plan:
-      - aggregate:
-        - get: gpos_github_release
-          params:
-            globs:
-              - bin_gpos_centos5_release.tar.gz
-          passed:
-            - build_with_orca
-        - get: orca_github_release
-          params:
-            globs:
-              - bin_orca_centos5_release.tar.gz
-          passed:
-            - build_with_orca
-        - get: bin_xerces
-          passed:
-            - build_with_orca
-        - get: bin_gpdb
-          passed:
-            - build_with_orca
-        - get: gpdb_src
-          trigger: true
-          passed:
-            - build_with_orca
-          params:
-            submodules: none
-      - task: icg
-        file: gpdb_src/concourse/test_with_orca.yml
-        input_mapping:
-          bin_orca: orca_github_release
-          bin_gpos: gpos_github_release
-        timeout: 2h30m
-        on_failure:
-          aggregate:
-          - put: regression_diffs
-            params:
-              from: icg_output/regression.diffs
-          - put: regression_out
-            params:
-              from: icg_output/regression.out

concourse/pr_pipeline.yml

-resource_types:
-- name: pull-request
-  type: docker-image
-  source:
-    repository: pivotaldata/pr-resource
-
-resources:
-  - name: gpdb_pr
-    type: pull-request
-    source:
-      repo: greenplum-db/gpdb
-      access_token: {{gpdb_access_token}}
-
-  - name: gpos_github_release
-    type: github-release
-    source:
-      user: greenplum-db
-      repository: gpos
-      access_token: {{gposbot_access_token}}
-
-  - name: orca_github_release
-    type: github-release
-    source:
-      user: greenplum-db
-      repository: gporca
-      access_token: {{gporcabot_access_token}}
-
-  - name: bin_xerces
-    type: s3
-    source:
-      access_key_id: {{aws-access-key-id}}
-      bucket: {{bucket-name}}
-      region_name: {{aws-region}}
-      secret_access_key: {{aws-secret-access-key}}
-      versioned_file: bin_xerces_centos5.tar.gz
-
-  - name: bin_gpdb
-    type: s3
-    source:
-      access_key_id: {{aws-access-key-id}}
-      bucket: gporca-concourse-bucket
-      region_name: {{aws-region}}
-      secret_access_key: {{aws-secret-access-key}}
-      versioned_file: bin_gpdb_with_orca_centos6.tar.gz
-
-  - name: regression_diffs
-    type: s3
-    source:
-       access_key_id: {{aws-access-key-id}}
-       bucket: {{bucket-name}}
-       region_name: {{aws-region}}
-       secret_access_key: {{aws-secret-access-key}}
-       versioned_file: regression.diffs
-
-  - name: regression_out
-    type: s3
-    source:
-       access_key_id: {{aws-access-key-id}}
-       bucket: {{bucket-name}}
-       region_name: {{aws-region}}
-       secret_access_key: {{aws-secret-access-key}}
-       versioned_file: regression.out
-
-
-jobs:
-  - name: build_with_orca
-    plan:
-      - aggregate:
-        - get: gpos_github_release
-          params:
-            globs:
-              - bin_gpos_centos5_release.tar.gz
-        - get: orca_github_release
-          params:
-            globs:
-              - bin_orca_centos5_release.tar.gz
-        - get: bin_xerces
-        - get: gpdb_pr
-          trigger: true
-      # update the PR status as pending
-      - put: gpdb_pr
-        params:
-          path: gpdb_pr
-          status: pending
-      - task: build_with_orca
-        file: gpdb_pr/concourse/build_with_orca.yml
-        input_mapping:
-          gpdb_src: gpdb_pr
-          bin_orca: orca_github_release
-          bin_gpos: gpos_github_release
-        on_success:
-          # update the PR status as pending
-          put: gpdb_pr
-          params:
-            path: gpdb_pr
-            status: pending
-        on_failure:
-          put: gpdb_pr
-          params:
-            path: gpdb_pr
-            status: failure
-      - task: package_tarball
-        file: gpdb_pr/concourse/package_tarball.yml
-        input_mapping:
-          gpdb_src: gpdb_pr
-        on_success:
-          put: gpdb_pr
-          params:
-            path: gpdb_pr
-            status: pending
-        on_failure:
-          put: gpdb_pr
-          params:
-            path: gpdb_pr
-            status: failure
-      - put: bin_gpdb
-        params:
-          file: package_tarball/bin_gpdb.tar.gz
-
-
-  - name: icg
-    plan:
-      - aggregate:
-        - get: gpos_github_release
-          params:
-            globs:
-              - bin_gpos_centos5_release.tar.gz
-          passed:
-            - build_with_orca
-        - get: orca_github_release
-          params:
-            globs:
-              - bin_orca_centos5_release.tar.gz
-          passed:
-            - build_with_orca
-        - get: bin_xerces
-          passed:
-            - build_with_orca
-        - get: bin_gpdb
-          passed:
-            - build_with_orca
-        - get: gpdb_pr
-          trigger: true
-          passed:
-            - build_with_orca
-      - put: gpdb_pr
-        params:
-          path: gpdb_pr
-          status: pending
-      - task: icg
-        file: gpdb_pr/concourse/test_with_orca.yml
-        input_mapping:
-          gpdb_src: gpdb_pr
-          bin_orca: orca_github_release
-          bin_gpos: gpos_github_release
-        timeout: 2h30m
-        on_success:
-          put: gpdb_pr
-          params:
-            path: gpdb_pr
-            status: success
-        on_failure:
-          aggregate:
-          - put: regression_diffs
-            params:
-              from: icg_output/regression.diffs
-          - put: regression_out
-            params:
-              from: icg_output/regression.out
-          - put: gpdb_pr
-            params:
-              path: gpdb_pr
-              status: failure

concourse/prepare_orca_dependencies.bash

-#!/bin/bash
-
-set -u -e -x
-
-IVYFILE="gpdb_src/gpAux/releng/make/dependencies/ivy.xml"
-PLATFORM="centos5"
-BUILDTYPE="release"
-
-setup_gpos() {
-    gposdir=$1
-    gposver=$(grep '\"libgpos\"' ${IVYFILE} | sed 's/.*rev="\([^"]*\)".*/\1/')
-    wget -P $gposdir https://github.com/greenplum-db/gpos/releases/download/v$gposver/bin_gpos_${PLATFORM}_${BUILDTYPE}.tar.gz
-}
-setup_gporca() {
-    gporcadir=$1
-    gporcaver=$(grep '\"optimizer\"' ${IVYFILE} | sed 's/.*rev="\([^"]*\)".*/\1/')
-    wget -P $gporcadir https://github.com/greenplum-db/gporca/releases/download/v$gporcaver/bin_orca_${PLATFORM}_${BUILDTYPE}.tar.gz
-}
-
-main() {
-    setup_gpos bin_gpos
-    setup_gporca bin_orca
-}
-
-main "$@"

concourse/prepare_orca_dependencies.yml

-platform: linux
-image: docker:///pivotaldata/gpdb-devel-gcc5#centos6
-inputs:
-  - name: gpdb_src
-outputs:
-  - name: bin_gpos
-  - name: bin_orca
-run:
-  path: gpdb_src/concourse/prepare_orca_dependencies.bash

concourse/provision.yml

-platform: linux
-image: docker:///pivotaldata/gpdb-devel-gcc5#perf
-inputs:
-  - name: gpdb_src
-outputs:
-  - name: instance_ids
-run:
-  path: /bin/bash
-  args:
-    - -l
-    - -c
-    - gpdb_src/concourse/scripts/perf/provision.sh

concourse/scripts/perf/common.sh

-log() {
-  echo -e "$@"
-}
-
-prepare_for_ssh() {
-  local IP=$1
-  local KEYFILE="${AWS_KEYPAIR}.pem"
-
-  log 'Preparing $IP for SSH'
-
-  if [[ ! -f "${KEYFILE}" ]]; then
-    echo "${AWS_KEYPAIR_VALUE}" > "${KEYFILE}"
-    chmod 0600 "${KEYFILE}"
-  fi
-
-  mkdir -p ~/.ssh
-  ssh-keyscan $IP >> ~/.ssh/known_hosts
-}
-
-remote_push() {
-  local IP=$1
-  local FROM=$2
-  local TO=$3
-  local KEYFILE="${AWS_KEYPAIR}.pem"
-
-  rsync -v --progress -e "ssh -i ${KEYFILE}" $FROM ${SSH_USER}@${IP}:$TO
-}
-
-remote_run() {
-  local IP="$1"
-  local KEYFILE="${AWS_KEYPAIR}.pem"
-
-  ssh -i "${KEYFILE}" -t -t ${SSH_USER}@${IP} "${@:2}"
-}
-
-remote_run_script() {
-  local IP="$1"
-  local USER="$2"
-  local EXPORTS="$3"
-  local KEYFILE="${AWS_KEYPAIR}.pem"
-
-  if [ -z "$EXPORTS" ]; then
-    EXPORTS=echo
-  fi
-
-  local SCRIPT_BODY=$(printf "%q" "${EXPORTS}; cd ~${USER}; $(cat $4)")
-
-  ssh -i "${KEYFILE}" -t -t ${SSH_USER}@${IP} "sudo -u ${USER} bash -c ${SCRIPT_BODY}"
-}
-
-
-error() {
-  echo >&2 "$@"
-  exit 1
-}
-
-check_config() {
-  log "Configuration"
-  log "=============================="
-
-  log "AWS_REGION=${AWS_REGION}"
-  export EC2_URL="https://ec2.${AWS_REGION}.amazonaws.com"
-
-  log "EC2_URL=${EC2_URL}"
-  log "AMI=${AMI}"
-  log "INSTANCE_TYPE=${INSTANCE_TYPE}"
-  log "INSTANCE_NAME=${INSTANCE_NAME}"
-  log "SUBNET_ID=${SUBNET_ID}"
-  log "SECURITY_GROUP_ID=${SECURITY_GROUP_ID}"
-  log "SSH_USER=${SSH_USER}"
-  log "TENANCY=${TENANCY}"
-
-  log "WAIT_SECS=${WAIT_SECS}"
-  export WAIT=${WAIT_SECS}
-
-  log "RETRIES=${RETRIES}"
-
-  log "=============================="
-}
-
-wait_until_sshable() {
-  IP=$1
-
-  local i
-  for i in $(seq "${RETRIES}"); do
-    if nc -z -v "${IP}" 22; then
-      return
-    fi
-    sleep $WAIT
-  done
-
-  error "Timed out waiting for ${IP} to come online."
-}
-
-wait_until_status() {
-  local STATUS=$1
-
-  log "Waiting for status: ${STATUS}"
-
-  local N=0
-  until [[ $N -ge $RETRIES ]]; do
-    local COUNT=$(
-      ec2-describe-instances --show-empty-fields ${INSTANCE_IDS[*]} |
-      grep INSTANCE |
-      cut -f6 |
-      grep -c ${STATUS}
-    ) || true
-
-    log "${COUNT} of ${#INSTANCE_IDS[@]} instances ${STATUS}"
-
-    [[ "$COUNT" -eq "${#INSTANCE_IDS[@]}" ]] && break
-
-    N=$(($N+1))
-    sleep $WAIT
-  done
-
-  if [[ $N -ge $RETRIES ]]; then
-    error "Timed out waiting for instances to reach status: ${STATUS}"
-  fi
-}
-
-wait_until_check_ok() {
-  log "Waiting for instances to pass status checks"
-
-  local N=0
-  until [[ $N -ge $RETRIES ]]; do
-    local COUNT
-
-    COUNT=$(
-      ec2-describe-instance-status --show-empty-fields ${INSTANCE_IDS[*]} |
-      grep -e "\bINSTANCE\b" |
-      cut -f6,7 |
-      xargs -n 2 |
-      grep -c "ok ok"
-    ) || true
-
-    log "${COUNT} of ${#INSTANCE_IDS[@]} instances pass status checks"
-
-    [[ "$COUNT" -eq "${#INSTANCE_IDS[@]}" ]] && break
-
-    COUNT=$(
-      ec2-describe-instance-status --show-empty-fields ${INSTANCE_IDS[*]} |
-      grep -e "\bINSTANCE\b" |
-      cut -f6,7 |
-      xargs -n 2 |
-      grep -c "impaired"
-    ) || true
-
-    if [[ "$COUNT" -gt 0 ]]; then
-      error "${COUNT} of ${#INSTANCE_IDS[@]} failed to pass status checks"
-    fi
-
-    N=$(($N+1))
-    sleep $WAIT
-  done
-
-  if [[ $N -ge $RETRIES ]]; then
-    error "Timed out waiting for instances to pass status checks"
-  fi
-}

concourse/scripts/perf/install_gpdb.sh

-#!/bin/bash
-
-set -e -u
-
-function more() {
-  cat $@ > /dev/null
-}
-
-main() {
-  if [[ -z ${VERIFY:-} ]]; then
-    apply
-  else
-    verify
-  fi
-}
-
-apply() {
-  export -f more
-  local MAGIC="yes\n\nyes\nyes"
-  unzip -o /tmp/${INSTALLER_NAME}.zip
-  echo -e ${MAGIC} | ./${INSTALLER_NAME}.bin
-  unset -f more
-}
-
-verify() {
-  # assert the file /usr/local/greenplum-db/greenplum_path exists
-  echo || [ -f /usr/local/greenplum-db/greenplum_path.sh ]
-}
-
-main "$@"

concourse/scripts/perf/prepare_kernel.sh

-#!/bin/bash
-
-set -e -u
-
-SYSCTL_HEAD="# BEGIN GENERATED CONTENT"
-SYSCTL_TAIL="# END GENERATED CONTENT"
-SYSCTL="$SYSCTL_HEAD
-kernel.hostname = ${HOST}
-kernel.shmmax = 500000000
-kernel.shmmni = 4096
-kernel.shmall = 4000000000
-kernel.sem = 250 512000 100 2048
-kernel.sysrq = 1
-kernel.core_uses_pid = 1
-kernel.msgmnb = 65536
-kernel.msgmax = 65536
-kernel.msgmni = 2048
-net.ipv4.tcp_syncookies = 1
-net.ipv4.ip_forward = 0
-net.ipv4.conf.default.accept_source_route = 0
-net.ipv4.tcp_tw_recycle = 1
-net.ipv4.tcp_max_syn_backlog = 4096
-net.ipv4.conf.all.arp_filter = 1
-net.ipv4.ip_local_port_range = 1025 65535
-net.ipv6.conf.all.disable_ipv6 = 1
-net.ipv6.conf.default.disable_ipv6 = 1
-net.core.netdev_max_backlog = 10000
-net.core.rmem_max = 2097152
-net.core.wmem_max = 2097152
-vm.overcommit_memory = 2
-vm.overcommit_ratio = 100
-$SYSCTL_TAIL"
-
-KERNEL="elevator=deadline transparent_hugepage=never"
-
-main() {
-  if [[ -z ${VERIFY:-} ]]; then
-    apply
-  else
-    verify
-  fi
-}
-
-apply() {
-  echo "Configuring Kernel"
-
-  sed -i -e "/$SYSCTL_HEAD/,/$SYSCTL_TAIL/d" /etc/sysctl.conf
-  echo "$SYSCTL" >> /etc/sysctl.conf  # Settings for next reboot
-  echo "$SYSCTL" | /sbin/sysctl -p -  # Current settings
-
-  sed -i -r "s/kernel(.+)/kernel\1 $KERNEL/" /boot/grub/grub.conf
-  echo never > /sys/kernel/mm/transparent_hugepage/enabled
-  for BLOCKDEV in /sys/block/*/queue/scheduler; do
-    echo deadline > "$BLOCKDEV"
-  done
-
-  # This needs to go into the AMI: Needed for starting GPDB
-  yum install -y ed
-
-  echo "Kernel configuration done."
-}
-
-verify() {
-  echo "Verifying changed sysctl params"
-  sysctl $(echo "$SYSCTL" | sed -e '/^#/d' | cut -f1 -d ' ')
-}
-
-main "$@"

concourse/scripts/perf/prepare_network.sh

-#!/bin/bash
-
-set -e
-
-HOSTFILE_HEAD="# BEGIN GENERATED CONTENT"
-HOSTFILE_TAIL="# END GENERATED CONTENT"
-
-main() {
-  if [[ -z ${VERIFY:-} ]]; then
-    apply
-  else
-    verify
-  fi
-}
-
-apply() {
-  echo "Configuring Network"
-
-  sed -i -e "/$HOSTFILE_HEAD/,/$HOSTFILE_TAIL/d" /etc/hosts
-  echo "$HOSTFILE_HEAD" >> /etc/hosts
-  echo "${PRIVATE_IP} ${HOST}" >> /etc/hosts
-  echo "$HOSTFILE_TAIL" >> /etc/hosts
-
-  update_known_hosts
-
-  echo "Configuration Done"
-}
-
-verify() {
-  echo "Verifying network configuration"
-  cat /etc/hosts
-  cat ~/.ssh/known_hosts
-}
-
-update_known_hosts() {
-
-    for U in root gpadmin; do
-      su $U -c "ssh-keygen -R ${HOST}" || true
-
-      su $U -c "ssh-keyscan ${HOST} >> ~/.ssh/known_hosts"
-
-      su $U -c "rm -f ~/.ssh/known_hosts.old"
-    done
-}
-
-main "$@"

concourse/scripts/perf/prepare_security.sh

-#!/bin/bash
-
-set -u
-
-SELINUX="
-SELINUX=disabled
-"
-
-LIMITS="
-* soft nofile 65536
-* hard nofile 65536
-* soft nproc 131072
-* hard nproc 131072
-"
-
-main() {
-  if [[ -z ${VERIFY:-} ]]; then
-    apply
-  else
-    verify
-  fi
-}
-
-apply() {
-  echo "Configuring Security"
-
-  echo "$LIMITS" > /etc/security/limits.d/99_overrides.conf
-  echo "$SELINUX" > /etc/selinux/config
-
-  disable_iptables
-
-  create_gpadmin
-
-  generate_keys
-
-  ensure_connectivity
-
-  echo "Configuration Done"
-}
-
-disable_iptables() {
-  /sbin/service iptables stop
-  /sbin/chkconfig iptables off
-}
-
-create_gpadmin() {
-  getent passwd gpadmin || useradd -m gpadmin
-}
-
-generate_keys() {
-  for U in root gpadmin; do
-    su $U -c "mkdir -p ~${U}/.ssh"
-    su $U -c "[[ -f ~${U}/.ssh/id_rsa ]] || ssh-keygen -N '' -f ~${U}/.ssh/id_rsa"
-
-    su $U -c "touch ~${U}/.ssh/authorized_keys"
-    su $U -c "chmod 600 ~${U}/.ssh/authorized_keys"
-  done
-}
-
-ensure_connectivity() {
-  for U in root gpadmin; do
-    su $U -c "cat ~$U/.ssh/id_rsa.pub >> ~$U/.ssh/authorized_keys"
-  done
-}
-
-verify() {
-  echo "Running ulimit to verify changes to limits"
-  ulimit -a
-}
-
-main "$@"

concourse/scripts/perf/provision.sh

-#!/bin/bash
-
-set -e -u
-
-source $(dirname "$0")/common.sh
-
-main() {
-  check_config
-  check_tools
-
-  run_instances
-  print_addresses
-
-  prepare_instances
-
-  log "Provision complete\n\n"
-}
-
-prepare_instances() {
-  local SCRIPTS=(prepare_kernel.sh prepare_security.sh prepare_network.sh)
-
-  local PRIVATE_IP=$(ec2-describe-instances --show-empty-fields ${INSTANCE_IDS} | grep INSTANCE | cut -f18) # Extract private IP
-  local exports="HOST=perfhost PRIVATE_IP=${PRIVATE_IP}"
-  for SCRIPT in "${SCRIPTS[@]}"; do
-    remote_run_script ${PRIVATE_IP} root "${exports}" "$(dirname "$0")/${SCRIPT}"
-  done
-
-  log "Restarting instance to apply all settings..."
-  ec2-reboot-instances ${INSTANCE_IDS} && sleep 30
-  wait_until_sshable ${PRIVATE_IP}
-
-  local exports+=" VERIFY=1"
-  for SCRIPT in "${SCRIPTS[@]}"; do
-    remote_run_script ${PRIVATE_IP} root "${exports}" "$(dirname "$0")/${SCRIPT}"
-  done
-}
-
-check_tools() {
-  if ! command -v ec2-run-instances >/dev/null 2>&1; then
-    error "Amazon EC2 API Tools not installed (see http://aws.amazon.com/developertools/351)"
-  fi
-}
-
-
-run_instances() {
-  log "Starting instances"
-
-  # If the instances contain any ephemeral volumes, this step is required to
-  # set up device mappings. The number of ephemeral volumes depends on the
-  # specific instance type. This does not fail if there are less than 24 such
-  # volumes.
-  BLOCK_DEV=(/dev/xvd{b..y})
-  EPHEMERAL=(ephemeral{0..23})
-  MAPPINGS=""
-  for I in $(seq 0 23); do
-    MAPPINGS+=" --block-device-mapping \"${BLOCK_DEV[$I]}=${EPHEMERAL[$I]}\""
-  done
-
-  INSTANCE_IDS=($(
-    ec2-run-instances ${AMI} \
-      -n 1 \
-      --tenancy ${TENANCY} \
-      --show-empty-fields \
-      -k ${AWS_KEYPAIR} \
-      --group ${SECURITY_GROUP_ID} \
-      --subnet ${SUBNET_ID} \
-      --instance-type ${INSTANCE_TYPE} \
-      ${MAPPINGS} |
-    grep INSTANCE |
-    cut -f2
-  ))
-	echo ${INSTANCE_IDS} > instance_ids/instance_ids.txt
-
-  log "Created instances: ${INSTANCE_IDS[*]}"
-
-  local INSTANCE_ID=${INSTANCE_IDS}
-  ec2-create-tags ${INSTANCE_ID} -t Name=${INSTANCE_NAME}
-
-  wait_until_status "running"
-  wait_until_check_ok
-
-  local PRIVATE_IP=$(ec2-describe-instances --show-empty-fields ${INSTANCE_IDS} | grep INSTANCE | cut -f18) # Extract private IP
-  prepare_for_ssh ${PRIVATE_IP}
-}
-
-print_addresses() {
-  log "INSTANCE\tPRIVATE IP"
-  ec2-describe-instances --show-empty-fields ${INSTANCE_IDS[*]} | grep INSTANCE | cut -f2,18
-  log ""
-}
-
-main "$@"

concourse/scripts/perf/setup_gpdb_with_installer.sh

-#!/bin/bash
-
-set -e -u
-
-source $(dirname "$0")/common.sh
-
-main() {
-  check_config
-
-  local INSTANCE_IDS=($(cat instance_ids/instance_ids.txt))
-  local PRIVATE_IP=$(ec2-describe-instances --show-empty-fields ${INSTANCE_IDS} | grep INSTANCE | cut -f18) # Extract private IP
-
-  prepare_for_ssh ${PRIVATE_IP}
-  
-  local INSTALLER_PATH="$(ls gpdb_installer/*.zip)"
-  local INSTALLER_NAME="$(basename ${INSTALLER_PATH} .zip)"
-  log "Found installer : ${INSTALLER_PATH}"
-
-  log "Copying the ${INSTALLER_PATH} to ${PRIVATE_IP}"
-  remote_push ${PRIVATE_IP} ${INSTALLER_PATH} /tmp
-
-  log "Running the installer remotely..."
-  remote_run_script ${PRIVATE_IP} root "INSTALLER_NAME=${INSTALLER_NAME}" $(dirname "$0")/install_gpdb.sh 
-  remote_run_script ${PRIVATE_IP} root "INSTALLER_NAME=${INSTALLER_NAME} VERIFY=1" $(dirname "$0")/install_gpdb.sh 
-
-  log "Starting GPDB cluster"
-  remote_run_script ${PRIVATE_IP} gpadmin "" $(dirname "$0")/start_gpdb.sh 
-  remote_run_script ${PRIVATE_IP} gpadmin "VERIFY=1" $(dirname "$0")/start_gpdb.sh 
-
-  log "GPDB Installation complete!"
-}
-
-main "$@"

concourse/scripts/perf/start_gpdb.sh

-#!/bin/bash
-
-set -e
-
-main() {
-  if [[ -z ${VERIFY:-} ]]; then
-    apply
-  else
-    verify
-  fi
-}
-
-apply() {
-  echo "source /usr/local/greenplum-db/greenplum_path.sh" >> ~gpadmin/.bashrc
-  source /usr/local/greenplum-db/greenplum_path.sh
-
-  local MASTER_HOSTNAME=perfhost
-  local SEGMENT_HOSTS=perfhost
-  local SEGMENTS_PER_HOST=3
-  local MASTER_DIRECTORY=~gpadmin/master
-  local SEGMENT_DIRECTORY=(~gpadmin/seg0 ~gpadmin/seg1 ~gpadmin/seg2)
-
-  echo "Preparing ~gpadmin/gpconfigs"
-
-  mkdir -p ~gpadmin/gpconfigs
-  cp /usr/local/greenplum-db/docs/cli_help/gpconfigs/gpinitsystem_config ~gpadmin/gpconfigs/
-  echo "$SEGMENT_HOSTS" > ~gpadmin/gpconfigs/hostfile_gpinitsystem
-
-  sed -i -r "s|#MACHINE_LIST_FILE|MACHINE_LIST_FILE|" ~gpadmin/gpconfigs/gpinitsystem_config
-  sed -i -r "s|MASTER_HOSTNAME=mdw|MASTER_HOSTNAME=${MASTER_HOSTNAME}|" ~gpadmin/gpconfigs/gpinitsystem_config
-  sed -i -r "s|MASTER_DIRECTORY=.*|MASTER_DIRECTORY=${MASTER_DIRECTORY}|" ~gpadmin/gpconfigs/gpinitsystem_config
-  sed -i -r "s| DATA_DIRECTORY=\(.*\)| DATA_DIRECTORY\=\(${SEGMENT_DIRECTORY[*]}\)|" ~gpadmin/gpconfigs/gpinitsystem_config
-
-  echo "Creating master and segment data directories"
-  mkdir -p ${MASTER_DIRECTORY} ${SEGMENT_DIRECTORY[*]}
-
-  echo "Starting GPDB"
-  gpinitsystem -a -c ~gpadmin/gpconfigs/gpinitsystem_config || [[ $? -lt 2 ]]
-}
-
-verify() {
-  source /usr/local/greenplum-db/greenplum_path.sh
-  psql template1 -c 'SELECT version();'
-  psql template1 -c 'SELECT gp_opt_version();'
-  psql template1 -c 'SELECT * FROM gp_segment_configuration;'
-  psql template1 -c 'SET optimizer=ON'
-}
-
-
-main "$@"

concourse/scripts/perf/unprovision.sh

-#!/bin/bash
-
-set -e -u
-
-source $(dirname "$0")/common.sh
-
-main() {
-  INSTANCE_IDS=($(cat instance_ids/instance_ids.txt))
-
-  check_config
-
-  log "Deleting instaces : ${INSTANCE_IDS}"
-  ec2-terminate-instances ${INSTANCE_IDS}
-
-  wait_until_status "terminated"
-  log "Instances terminated successfully"
-}
-
-main "$@"

concourse/setup_gpdb_with_installer.yml

-platform: linux
-image: docker:///pivotaldata/gpdb-devel-gcc5#perf
-inputs:
-  - name: gpdb_src
-  - name: instance_ids
-  - name: gpdb_installer
-run:
-  path: /bin/bash
-  args:
-    - -l
-    - -c
-    - gpdb_src/concourse/scripts/perf/setup_gpdb_with_installer.sh

concourse/unprovision.yml

-platform: linux
-image: docker:///pivotaldata/gpdb-devel-gcc5#perf
-inputs:
-  - name: instance_ids
-  - name: gpdb_src
-run:
-  path: /bin/bash
-  args:
-    - -l
-    - -c
-    - gpdb_src/concourse/scripts/perf/unprovision.sh