Replace separate_qautils_files with remove_non_production_files

The task was originally separating NON_PRODUCTION_FILES.txt out of the
bin_gpdb tarball and putting them into the QAUTILS_TARBALL, and leaving
OUTPUT_TARBALL with the rest of the files.
However, QAUTILS_TARBALL is never used, the only important function is
to remove the NON_PRODUCTION_FILES.txt.

With this commit, we refactor the separate_qautils_files to only remove
files. Also, we introduce a new pattern for TDD'ing bash scripts by
using the MIT-licensed BATS framework.
Co-authored-by: NMark Sliva <msliva@pivotal.io>
Co-authored-by: NAmil Khanzada <akhanzada@pivotal.io>

concourse/pipelines/gpdb_opensource_release.yml

@@ -160,16 +160,13 @@ jobs:
     - get: bin_gpdb_centos6
       passed: [compile_gpdb_centos6]
     - get: gpdb6-centos6-build
-  # Even though we don't ship the QA utils, we still need to strip out non-shipping files
-  - task: separate_qautils_files_for_rc_centos6
-    file: gpdb_src/concourse/tasks/separate_qautils_files_for_rc.yml
+  - task: remove_non_production_files
+    file: gpdb_src/concourse/tasks/remove_non_production_files.yml
     image: gpdb6-centos6-build
     input_mapping:
       bin_gpdb: bin_gpdb_centos6
     output_mapping:
       rc_bin_gpdb: rc_bin_gpdb_rhel6
-    params:
-      QAUTILS_TARBALL: "rc_bin_gpdb/QAUtils-rhel6-x86_64.tar.gz"
   - put: bin_gpdb_centos6
     params:
       file: rc_bin_gpdb_rhel6/bin_gpdb.tar.gz
@@ -183,16 +180,13 @@ jobs:
     - get: bin_gpdb_centos7
       passed: [compile_gpdb_centos7]
     - get: gpdb6-centos7-build
-  # Even though we don't ship the QA utils, we still need to strip out non-shipping files
-  - task: separate_qautils_files_for_rc_centos7
-    file: gpdb_src/concourse/tasks/separate_qautils_files_for_rc.yml
+  - task: remove_non_production_files
+    file: gpdb_src/concourse/tasks/remove_non_production_files.yml
     image: gpdb6-centos7-build
     input_mapping:
       bin_gpdb: bin_gpdb_centos7
     output_mapping:
       rc_bin_gpdb: rc_bin_gpdb_rhel7
-    params:
-      QAUTILS_TARBALL: "rc_bin_gpdb/QAUtils-rhel7-x86_64.tar.gz"
   - put: bin_gpdb_centos7
     params:
       file: rc_bin_gpdb_rhel7/bin_gpdb.tar.gz
@@ -206,16 +200,13 @@ jobs:
     - get: bin_gpdb_ubuntu18.04
       passed: [compile_gpdb_ubuntu18.04]
     - get: gpdb6-ubuntu18.04-build
-  # Even though we don't ship the QA utils, we still need to strip out non-shipping files
-  - task: separate_qautils_files_for_rc_ubuntu18.04
-    file: gpdb_src/concourse/tasks/separate_qautils_files_for_rc.yml
+  - task: remove_non_production_files
+    file: gpdb_src/concourse/tasks/remove_non_production_files.yml
     image: gpdb6-ubuntu18.04-build
     input_mapping:
       bin_gpdb: bin_gpdb_ubuntu18.04
     output_mapping:
       rc_bin_gpdb: rc_bin_gpdb_ubuntu18.04
-    params:
-      QAUTILS_TARBALL: "rc_bin_gpdb/QAUtils-ubuntu18.04-amd64.tar.gz"
   - put: bin_gpdb_ubuntu18.04
     params:
       file: rc_bin_gpdb_ubuntu18.04/bin_gpdb.tar.gz
@@ -289,4 +280,4 @@ jobs:
       body: release_artifacts/body
       globs:
         - release_artifacts/*.tar.gz
-        - release_artifacts/*.zip
\ No newline at end of file
+        - release_artifacts/*.zip

concourse/scripts/remove_non_production_files.bats

+#!/usr/bin/env bats
+
+setup() {
+  mkdir test_dir
+  cd test_dir
+
+  export OUTPUT_TARBALL="out.tar.gz"
+
+  touch a b c
+
+  tar --create --gzip --file a.tar.gz a
+  tar czf ab.tar.gz a b
+  tar czf abc.tar.gz a b c
+
+  echo -e ""        > empty-NON_PRODUCTION_FILES.txt
+  echo -e "a"       > a-NON_PRODUCTION_FILES.txt
+  echo -e "a\nb"    > ab-NON_PRODUCTION_FILES.txt
+  echo -e "a\nb\nc" > abc-NON_PRODUCTION_FILES.txt
+}
+
+teardown() {
+  cd ..
+  rm -rf test_dir
+}
+
+comm_tarballs() {
+  comm -3 <(tar -tf "$1") <(tar -tf "$2") | tr -d '[:space:]'
+}
+
+@test "test comm_tarballs with same file" {
+  run comm_tarballs a.tar.gz a.tar.gz
+  [ "$output" = "" ]
+}
+
+@test "test comm_tarballs with different file" {
+  run comm_tarballs a.tar.gz ab.tar.gz
+  [ "$output" = "b" ]
+}
+
+main() {
+  ../scripts/remove_non_production_files.sh
+}
+
+@test "it fails if any variables are unbound" {
+  run main
+  [ "$status" -ne 0 ]
+}
+
+@test "it prints out the removed files" {
+  export INPUT_TARBALL="abc.tar.gz"
+  export NON_PRODUCTION_FILES="ab-NON_PRODUCTION_FILES.txt"
+
+  run main
+  [ "$status" -eq 0 ]
+  [[ "$output" =~ $'a\nb' ]]
+}
+
+@test "it works when there are no files to remove" {
+  export INPUT_TARBALL="a.tar.gz"
+  export NON_PRODUCTION_FILES="empty-NON_PRODUCTION_FILES.txt"
+
+  run main
+  [ "$status" -eq 0 ]
+
+  run comm_tarballs "${INPUT_TARBALL}" "${OUTPUT_TARBALL}"
+  [ "$output" = "" ]
+}
+
+@test "it works when we remove a from ab" {
+  export INPUT_TARBALL="ab.tar.gz"
+  export NON_PRODUCTION_FILES="a-NON_PRODUCTION_FILES.txt"
+
+  run main
+  [ "$status" -eq 0 ]
+
+  run comm_tarballs "${INPUT_TARBALL}" "${OUTPUT_TARBALL}"
+  [ "$output" = "a" ]
+}
+
+@test "it works when we remove ab from abc" {
+  export INPUT_TARBALL="abc.tar.gz"
+  export NON_PRODUCTION_FILES="ab-NON_PRODUCTION_FILES.txt"
+
+  run main
+  [ "$status" -eq 0 ]
+
+  run comm_tarballs "${INPUT_TARBALL}" "${OUTPUT_TARBALL}"
+  [ "$output" = "ab" ]
+}
+
+@test "it works when we remove abc from abc" {
+  export INPUT_TARBALL="abc.tar.gz"
+  export NON_PRODUCTION_FILES="abc-NON_PRODUCTION_FILES.txt"
+
+  run main
+  [ "$status" -ne 0 ]
+
+  grep "All files were removed so ${OUTPUT_TARBALL} cannot be created" <(echo "$output")
+  [ "$?" -eq 0 ]
+}
+
+@test "it fails when we remove ab from a" {
+  export INPUT_TARBALL="a.tar.gz"
+  export NON_PRODUCTION_FILES="ab-NON_PRODUCTION_FILES.txt"
+
+  run main
+  [ "$status" -ne 0 ]
+  [[ "$output" =~ "rm: b: No such file or directory" ]]
+}

concourse/scripts/remove_non_production_files.sh

+#!/bin/bash
+
+set -euxo pipefail
+
+main() {
+  local tmp_dir="$(mktemp -d)"
+  local wd=$(pwd)
+
+  tar zxf "${INPUT_TARBALL}" -C "${tmp_dir}"
+
+  cd "${tmp_dir}"
+  xargs rm -v < "${wd}/${NON_PRODUCTION_FILES}"
+  if ! tar czf "${wd}/${OUTPUT_TARBALL}" * ; then
+    echo "All files were removed so ${OUTPUT_TARBALL} cannot be created"
+    exit 1
+  fi
+}
+
+main "$@"

concourse/scripts/separate_qautils_files_for_rc.bash

-#!/bin/bash
-
-set -euo pipefail
-
-main() {
-  ABS_PATH_OUTPUT_TARBALL="$(pwd)/$OUTPUT_TARBALL"
-  ABS_QAUTILS_FILES="$(pwd)/$QAUTILS_FILES"
-  ABS_PATH_QAUTILS_TARBALL="$(pwd)/$QAUTILS_TARBALL"
-  QAUTILS_DIR="$(mktemp -d)"
-
-  INTERMEDIATE_PLACE="$(mktemp -d)"
-  tar zxf "$INPUT_TARBALL" -C "$INTERMEDIATE_PLACE"
-
-  pushd "$INTERMEDIATE_PLACE"
-    echo "Move files listed in $ABS_QAUTILS_FILES"
-    while read file; do
-      if [ -f "$file" ]; then
-	TARGET_QAUTILS_DIR="$QAUTILS_DIR"/`dirname $file`
-	echo "Moving $file to directory $TARGET_QAUTILS_DIR"
-	mkdir -p "$TARGET_QAUTILS_DIR"
-	mv "$file" "$TARGET_QAUTILS_DIR"
-      else
-	echo "File $file does not exists, skipping moving it"
-      fi
-    done < "$ABS_QAUTILS_FILES"
-    tar czf "$ABS_PATH_OUTPUT_TARBALL" *
-  popd
-
-  pushd "$QAUTILS_DIR"
-    tar czf "$ABS_PATH_QAUTILS_TARBALL" *
-  popd
-}
-
-main "$@"

concourse/tasks/separate_qautils_files_for_rc.yml → concourse/tasks/remove_non_production_files.yml

@@ -7,9 +7,8 @@ inputs:
 outputs:
   - name: rc_bin_gpdb
 run:
-  path: gpdb_src/concourse/scripts/separate_qautils_files_for_rc.bash
+  path: gpdb_src/concourse/scripts/remove_non_production_files.sh
 params:
-  QAUTILS_FILES: gpdb_src/gpAux/releng/NON_PRODUCTION_FILES.txt
+  NON_PRODUCTION_FILES: gpdb_src/gpAux/releng/NON_PRODUCTION_FILES.txt
   INPUT_TARBALL: bin_gpdb/bin_gpdb.tar.gz
   OUTPUT_TARBALL: rc_bin_gpdb/bin_gpdb.tar.gz
-  QAUTILS_TARBALL: rc_bin_gpdb/QAUtils-OS-x86_64.tar.gz