Unify backup/restore tests to work with Data Domain

This is part of the effort to unify our backup/restore tests into a
single suite.

* Adds infrastructure to setup DDBoost on the client and cleanup server
* after completion
* Adds tests for DDBoost specific options
* Adds test coverage from TINC suite that was not included in behave
Signed-off-by: NChris Hajas <chajas@pivotal.io>

gpMgmt/Makefile.behave

@@ -6,6 +6,7 @@ BEHAVE_BIN=$(PYTHONSRC_INSTALL)/bin/behave
 PYTHONSRC_INSTALL_SITE=$(shell find $(PYLIB_SRC_EXT)/install/lib -name "site-packages")
 PYTHONSRC_INSTALL_PYTHON_PATH=$(PYTHONPATH):$(PYTHONSRC_INSTALL_SITE)
 TAR?=$(shell which tar)
+PEXPECT_LIB=$(GPHOME)/bin/lib
 
 $(BEHAVE_BIN):
 	TAR=$(TAR) make -C bin $(BEHAVE_BIN)
@@ -15,9 +16,9 @@ $(BEHAVE_BIN):
 behave: $(BEHAVE_BIN)
 	@echo "Running behave on management scripts..."
 	@if [ -n """$(flags)""" ]; then \
-	        PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH):$(GPMGMT_SRC) python $(BEHAVE_BIN) $(GPMGMT_SRC)/test/behave/* -s -k $(flags) 2>&1 ; \
+		PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH):$(GPMGMT_SRC):$(PEXPECT_LIB) python $(BEHAVE_BIN) $(GPMGMT_SRC)/test/behave/* -s -k $(flags) 2>&1 ; \
 	elif [ -n """$(tags)""" ]; then \
-	        PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH):$(GPMGMT_SRC) python $(BEHAVE_BIN) $(GPMGMT_SRC)/test/behave/* -s -k --tags=$(tags) 2>&1 ; \
+		PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH):$(GPMGMT_SRC):$(PEXPECT_LIB) python $(BEHAVE_BIN) $(GPMGMT_SRC)/test/behave/* -s -k --tags=$(tags) 2>&1 ; \
 	else \
-	        echo "Please specify tags=tagname or flags=[behave flags]"; \
+		echo "Please specify tags=tagname or flags=[behave flags]"; \
 	fi

gpMgmt/test/behave/mgmt_utils/backups.feature

@@ -197,6 +197,7 @@ Feature: Validate command line arguments
         And gpcrondump should print "Invalid state file format" to stdout
 
     @nbupartIII
+    @ddpartIII
     Scenario: Increments File Check With Complicated Scenario
         Given the backup test is initialized with no backup files
         And database "bkdb2" is dropped and recreated
@@ -266,6 +267,7 @@ Feature: Validate command line arguments
         And gpcrondump should print "Dump type                                = Incremental" to stdout
 
     @nbupartIII
+    @ddpartIII
     Scenario: gpcrondump -G with Full timestamp
         Given the backup test is initialized with no backup files
         And there is a "heap" table "public.heap_table" in "bkdb" with data
@@ -280,7 +282,6 @@ Feature: Validate command line arguments
         Then gpcrondump should return a return code of 0
         And "global" file should be created under " "
 
-    @nbupartIII
     @valgrind
     Scenario: Valgrind test of gp_dump incremental
         Given the backup test is initialized with no backup files
@@ -330,7 +331,6 @@ Feature: Validate command line arguments
         Then gpcrondump should return a return code of 0
         And the user runs valgrind with "gp_dump_agent --gp-k 11111111111111_-1_1_ --gp-d /tmp --pre-data-schema-only bkdb --incremental --table-file=/tmp/dirty_hack.txt" and options " "
 
-    @nbupartIII
     @valgrind
     Scenario: Valgrind test of gp_dump_agent full with table file
         Given the backup test is initialized with no backup files
@@ -345,7 +345,6 @@ Feature: Validate command line arguments
         And the user runs valgrind with "gp_dump_agent --gp-k 11111111111111_-1_1_ --gp-d /tmp --pre-data-schema-only bkdb --table-file=/tmp/dirty_hack.txt" and options " "
 
     @valgrind
-    @nbupartIII
     Scenario: Valgrind test of gp_dump_agent incremental
         Given the backup test is initialized with no backup files
         And there is a "heap" table "public.heap_table" in "bkdb" with data
@@ -428,6 +427,7 @@ Feature: Validate command line arguments
         And the dump directory for the stored timestamp should exist
 
     @nbupartIII
+    @ddpartIII
     Scenario: Verify the gpcrondump -g option works with full backup
         Given the backup test is initialized with no backup files
         When the user runs "gpcrondump -a -x bkdb -g"
@@ -436,6 +436,7 @@ Feature: Validate command line arguments
         And config files should be backed up on all segments
 
     @nbupartIII
+    @ddpartIII
     Scenario: Verify the gpcrondump -g option works with incremental backup
         Given the backup test is initialized with no backup files
         When the user runs "gpcrondump -a -x bkdb"
@@ -446,6 +447,7 @@ Feature: Validate command line arguments
         And config files should be backed up on all segments
 
     @nbupartIII
+    @ddpartIII
     Scenario: Verify the gpcrondump history table works by default with full and incremental backups
         Given the backup test is initialized with no backup files
         And schema "testschema" exists in "bkdb"
@@ -464,6 +466,7 @@ Feature: Validate command line arguments
         And verify that the table "gpcrondump_history" in "bkdb" has dump info for the stored timestamp
 
     @nbupartIII
+    @ddpartIII
     Scenario: Verify the gpcrondump -H option should not create history table
         Given the backup test is initialized with no backup files
         And schema "testschema" exists in "bkdb"
@@ -479,6 +482,7 @@ Feature: Validate command line arguments
         And gpcrondump should print "-H option cannot be selected with -h option" to stdout
 
     @nbupartIII
+    @ddpartIII
     Scenario: Config files have the same timestamp as the backup set
         Given the backup test is initialized with no backup files
         And there is a "heap" table "public.heap_table" in "bkdb" with data
@@ -486,9 +490,10 @@ Feature: Validate command line arguments
         When the user runs "gpcrondump -a -x bkdb -g"
         And the timestamp from gpcrondump is stored
         Then gpcrondump should return a return code of 0
-        And verify that the config files are backed up with the stored timestamp
+        And config files should be backed up on all segments
 
     @nbupartIII
+    @ddpartIII
     Scenario Outline: Incremental Backup With column-inserts, inserts and oids options
         Given the backup test is initialized with no backup files
         When the user runs "gpcrondump -a --incremental -x bkdb <options>"
@@ -524,6 +529,7 @@ Feature: Validate command line arguments
         Then gpcrondump should return a return code of 0
 
     @nbupartIII
+    @ddpartIII
     Scenario: Full Backup with option -t and non-existant table
         Given the backup test is initialized with no backup files
         And there is a "heap" table "public.heap_table" in "bkdb" with data
@@ -693,17 +699,74 @@ Feature: Validate command line arguments
         Then verify the metadata dump file does not contain "ALTER TABLE * OWNER TO"
 
     @nbupartIII
-    Scenario: gpcrondump with -u, -G, and -g
+    Scenario: gpcrondump with -G and -g
         Given the backup test is initialized with no backup files
         And there is a "heap" table "public.heap_table" in "bkdb" with data
         And there is a "ao" table "public.ao_index_table" in "bkdb" with data
-        When the user runs "gpcrondump -a -x bkdb -G -g -u /tmp"
+        When the user runs "gpcrondump -a -x bkdb -G -g"
         And the timestamp from gpcrondump is stored
         Then gpcrondump should return a return code of 0
-        And "global" file should be created under "/tmp"
-        And config files should be backed up on all segments in directory "/tmp"
+        And "global" file should be created under " "
+        And config files should be backed up on all segments
+
+    Scenario: gpcrondump with -k (vacuum after backup)
+        Given the backup test is initialized with no backup files
+        And the user runs "psql -c 'vacuum;' bkdb"
+        Then store the vacuum timestamp for verification in database "bkdb"
+        When the user runs "gpcrondump -a -x bkdb -k"
+        Then gpcrondump should return a return code of 0
+        Then gpcrondump should print "Commencing post-dump vacuum" to stdout
+        And gpcrondump should print "Vacuum of bkdb completed without error" to stdout
+        And verify that vacuum has been run in database "bkdb"
+
+    Scenario: gpcrondump with -f when not enough disk space
+        Given the backup test is initialized with no backup files
+        When the user runs "gpcrondump -a -x bkdb -f 100"
+        Then gpcrondump should return a return code of 2
+        And gpcrondump should print "segment\(s\) failed disk space checks" to stdout
+
+    Scenario: gpcrondump with -B with 1 process
+        Given the backup test is initialized with no backup files
+        When the user runs "gpcrondump -a -x bkdb -B 1 -v"
+        Then gpcrondump should return a return code of 0
+        And gpcrondump should not print "\[worker1\] got a halt cmd" to stdout
+
+    Scenario: gpcrondump with -d with invalid master data directory
+        Given the backup test is initialized with no backup files
+        When the user runs "gpcrondump -a -x bkdb -d /tmp"
+        Then gpcrondump should return a return code of 2
+        And gpcrondump should print "gpcrondump failed.* No such file or directory" to stdout
+
+    Scenario: gpcrondump with -l to log to /tmp directory
+        Given the backup test is initialized with no backup files
+        When the user runs "gpcrondump -a -x no_exist -l /tmp"
+        Then gpcrondump should return a return code of 2
+        And the "gpcrondump" log file should exist under "/tmp"
+
+    @ddpartIII
+    @ddonly
+    Scenario: gpcrondump with -c on Data Domain
+        Given the backup test is initialized with no backup files
+        When the user runs "gpcrondump -a -x bkdb -K 20150101010101"
+        Then gpcrondump should return a return code of 0
+        And the full backup timestamp from gpcrondump is stored
+        When the user runs "gpcrondump -a -x bkdb -c"
+        Then gpcrondump should return a return code of 0
+        Then no dump files should be present on the data domain server
+
+    @ddpartIII
+    @ddonly
+    Scenario: gpcrondump with -o on Data Domain
+        Given the backup test is initialized with no backup files
+        When the user runs "gpcrondump -a -x bkdb -K 20150101010101"
+        Then gpcrondump should return a return code of 0
+        And the full backup timestamp from gpcrondump is stored
+        When the user runs "gpcrondump -a -x bkdb -o"
+        Then gpcrondump should return a return code of 0
+        Then no dump files should be present on the data domain server
 
     @nbupartIII
+    @ddpartIII
     Scenario: Out of Sync timestamp
         Given the backup test is initialized with no backup files
         And there is a "ao" table "public.ao_table" in "bkdb" with data
@@ -715,5 +778,11 @@ Feature: Validate command line arguments
         And gpcrondump should print "There is a future dated backup on the system preventing new backups" to stdout
 
     @nbupartIII
+    @ddpartIII
     Scenario: The test suite is torn down
         Given the backup test is initialized with no backup files
+
+    @ddonly
+    @ddboostsetup
+    Scenario: Cleanup DDBoost dump directories
+        Given the DDBoost dump directory is deleted

gpMgmt/test/behave/mgmt_utils/restores.feature

@@ -32,6 +32,7 @@ Feature: Validate command line arguments
         Then gpdbrestore should return a return code of 2
         And gpdbrestore should print "Name has an invalid character" to stdout
 
+    @ddpartIII
     Scenario: gpdbrestore -b with Full timestamp
         Given the backup test is initialized with no backup files
         And there is a "ao" table "public.ao_index_table" in "bkdb" with data
@@ -82,3 +83,18 @@ Feature: Validate command line arguments
         Then gpdbrestore should return a return code of 2
         And gpdbrestore should print "-u cannot be used with DDBoost parameters" to stdout
 
+    Scenario: gpdbrestore with -d with invalid master data directory
+        When the user runs "gpdbrestore -a -t 20140101010101 -d /tmp"
+        Then gpdbrestore should return a return code of 2
+        And gpdbrestore should print "gpdbrestore failed.* No such file or directory" to stdout
+
+    Scenario: gpdbrestore with -l to log to /tmp directory
+        Given the backup test is initialized with no backup files
+        When the user runs "gpdbrestore -l /tmp"
+        Then gpdbrestore should return a return code of 2
+        And the "gpdbrestore" log file should exist under "/tmp"
+
+    @ddonly
+    @ddboostsetup
+    Scenario: Cleanup DDBoost dump directories
+        Given the DDBoost dump directory is deleted

gpMgmt/test/behave_utils/utils.py

@@ -1011,7 +1011,12 @@ def get_backup_dirs_for_hosts(dbname='template1'):
 def cleanup_backup_files(context, dbname, location=None):
     dir_map = get_backup_dirs_for_hosts(dbname)
     for host in dir_map:
-        if location:
+
+        if os.getenv('DDBOOST'):
+            ddboost_dir = context._root['ddboost_backupdir']
+            cmd_str = "ssh %s 'for DIR in %s; do if [ -d \"$DIR/%s\" ]; then rm -rf $DIR/%s $DIR/gpcrondump.pid; fi; done'"
+            cmd = cmd_str % (host, " ".join(dir_map[host]), ddboost_dir, ddboost_dir)
+        elif location:
             cmd_str = "ssh %s 'DIR=%s;if [ -d \"$DIR/db_dumps/\" ]; then rm -rf $DIR/db_dumps $DIR/gpcrondump.pid; fi'"
             cmd = cmd_str % (host, location)
         else: