Instantiate AO, CO, and last op state files with newline in backups (#2253)

gpddboost will fail to upload an empty (0 byte) file to Data Domain.
Now, in cases where we previously uploaded an empty file we insert a
newline to allow gpddboost to upload successfully. We also update the
logic on incremental backups to correctly parse these files.

Authors: Chris Hajas and Karen Huddleston

gpMgmt/bin/gppylib/operations/dump.py

@@ -230,6 +230,8 @@ def compare_metadata(old_pgstatoperations, cur_pgstatoperations):
 def get_pgstatlastoperations_dict(last_operations):
     last_operations_dict = {}
     for operation in last_operations:
+        if operation == '':
+            continue
         toks = operation.split(',')
         if len(toks) != 6:
             raise Exception('Wrong number of tokens in last_operation data for last backup: "%s"' % operation)
@@ -262,6 +264,8 @@ def get_last_dump_timestamp(context):
 def create_partition_dict(partition_list):
     table_dict = dict()
     for partition in partition_list:
+        if partition == '':
+            continue
         # As of version 4.3.8.0, gpcrondump supports spaces in schema name (field[0]) and table name (field[1])
         # so we explicitly split on "comma followed by space" instead of just commas since we can't strip() the string
         fields = partition.split(', ')
@@ -293,6 +297,8 @@ def get_filename_from_filetype(context, table_type, timestamp=None):
 def write_state_file(context, table_type, partition_list):
     filename = get_filename_from_filetype(context, table_type, None)
 
+    if not partition_list:
+        partition_list = ['\n']
     write_lines_to_file(filename, partition_list)
 
     if context.ddboost:
@@ -398,6 +404,8 @@ def write_partition_list_file(context, timestamp=None):
 
 def write_last_operation_file(context, rows):
     filename = context.generate_filename("last_operation")
+    if not rows:
+        rows = ['\n']
     write_lines_to_file(filename, rows)
 
     if context.ddboost:

gpMgmt/bin/gppylib/operations/test/unit/test_unit_dump.py

@@ -278,6 +278,18 @@ class DumpTestCase(unittest.TestCase):
             for i in range(len(part_list)):
                 self.assertEqual(call(part_list[i]+'\n'), result.write.call_args_list[i])
 
+    @patch('gppylib.operations.dump.get_filename_from_filetype', return_value='/tmp/db_dumps/20170413/gp_dump_20170413224743_ao_state_file')
+    def test_write_state_file_empty(self, mock1):
+        table_type = 'ao'
+        part_list = ['']
+        m = mock_open()
+        with patch('__builtin__.open', m, create=True):
+            write_state_file(self.context, table_type, part_list)
+            result = m()
+            self.assertEqual(1, len(result.write.call_args_list))
+            for i in range(len(part_list)):
+                self.assertEqual(call('\n'), result.write.call_args_list[i])
+
     @patch('gppylib.operations.dump.execute_sql', return_value=[['public', 'ao_table', 123, 'CREATE', 'table', '2012: 1'], ['testschema', 'co_table', 333, 'TRUNCATE', '', '2033 :1 - 111']])
     def test_get_last_operation_data_default(self, mock):
         output = get_last_operation_data(self.context)
@@ -367,7 +379,7 @@ class DumpTestCase(unittest.TestCase):
         self.assertEqual(result, expected_output)
 
     def test_create_partition_dict_empty(self):
-        partition_list = []
+        partition_list = ['']
         expected_output = {}
         result = create_partition_dict(partition_list)
         self.assertEqual(result, expected_output)
@@ -438,7 +450,7 @@ class DumpTestCase(unittest.TestCase):
         self.assertEqual(last_operations_dict, expected_output)
 
     def test_get_pgstatlastoperations_dict_empty(self):
-        last_operations = []
+        last_operations = ['']
         last_operations_dict = get_pgstatlastoperations_dict(last_operations)
         expected_output = {}
         self.assertEqual(last_operations_dict, expected_output)