Allow gpdbrestore to restore files from different dbids on datadomain (#4915)

This was supposed to be supported with the addition of content id to
filenames, but it was not working.

Added feature to gpddboost --readFile option to accept regular
expression in filename and search the data domain directory for a file
that matches. If a file exists with the matching content id, but a
different dbid, we will find it and be able to restore.
Co-authored-by: NChris Hajas <chajas@pivotal.io>
Co-authored-by: NKaren Huddleston <khuddleston@pivotal.io>

gpMgmt/test/behave/mgmt_utils/backup_and_restore_backups.feature

@@ -2014,3 +2014,17 @@ Feature: Validate command line arguments
         And the files in the backup sets are validated for the "local" storage unit
         And the files in the backup sets are validated for the "remote" storage unit
         And the backup sets on the "local" storage unit are deleted using gp_mfr
+
+    @ddpartIII
+    @ddonly
+    @skip_filename_compatibility
+    Scenario: 150 Full backup and restore with DataDomain to different dbids
+        Given the backup test is initialized with database "bkdb150"
+        And there is a "heap" table "public.heap_table" in "bkdb150" with data
+        And there is a "ao" table "public.ao_index_table" in "bkdb150" with data
+        And a backup file of tables "public.heap_table, public.ao_index_table" in "bkdb150" exists for validation
+        And all the data from "bkdb150" is saved for verification
+        When the user runs "gpcrondump -a -x bkdb150"
+        Then gpcrondump should return a return code of 0
+        And the timestamp from gpcrondump is stored
+        And the backup files on data domain are renamed with different dbids

gpMgmt/test/behave/mgmt_utils/backup_and_restore_restores.feature

@@ -1499,8 +1499,18 @@ Feature: Validate command line arguments
         And verify that the tuple count of all appendonly tables are consistent in "bkdb149"
         And the backup sets on the "remote" storage unit are deleted using gp_mfr
 
+    @ddpartIII
+    @ddonly
+    @skip_filename_compatibility
+    Scenario: 150 Full backup and restore with DataDomain to different dbids
+        Given the old timestamps are read from json
+        When the user runs gpdbrestore -e with the stored timestamp
+        And gpdbrestore should return a return code of 0
+        Then verify that the data of "2" tables in "bkdb150" is validated after restore
+        And verify that the tuple count of all appendonly tables are consistent in "bkdb150"
+
     @ddonly
     @ddboostsetup
-    Scenario: 150 Cleanup DDBoost dump directories
+    Scenario: 151 Cleanup DDBoost dump directories
         Given the DDBoost dump directory is deleted for the "local" storage unit
         Given the DDBoost dump directory is deleted for the "remote" storage unit

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -4878,6 +4878,38 @@ def store_timestamp_in_old_format(context, directory=None, prefix=""):
 
         run_command(context, change_report_file_content)
 
+@given('the backup files on data domain are renamed with different dbids')
+@when('the backup files on data domain are renamed with different dbids')
+@then('the backup files on data domain are renamed with different dbids')
+def impl(context):
+    rename_files_for_dbids_on_data_domain(context)
+
+def rename_files_for_dbids_on_data_domain(context):
+    gparray = GpArray.initFromCatalog(dbconn.DbURL())
+    primary_segs = [seg for seg in gparray.getSegDbList() if seg.isSegmentPrimary()]
+    num_segs = len(primary_segs)
+
+    if context.backup_timestamp is not None:
+        timestamp = context.backup_timestamp
+    else:
+        timestamp = context.full_backup_timestamp
+
+    ddboost_dir = os.path.join(context._root["ddboost_backupdir"], timestamp[0:8])
+    name_dict = {}
+    for ps in primary_segs:
+        segdbId = ps.getSegmentDbId()
+        segcid = ps.getSegmentContentId()
+        old_filename = "%s/gp_dump_%d_%d_%s.gz" % (ddboost_dir, segcid, segdbId, timestamp)
+        new_filename = "%s/gp_dump_%d_%d_%s.gz" % (ddboost_dir, segcid, segdbId+num_segs, timestamp)
+        name_dict[old_filename] = new_filename
+
+    for new_name, old_name in name_dict.iteritems():
+        rename_files_cmd = "gpddboost --rename --from-file=%s --to-file=%s" % (new_name, old_name)
+
+        run_command(context, rename_files_cmd)
+
+        if context.exception:
+            raise context.exception
 
 # todo this seems like it can only be a given or a when, not a "then"
 @then('the timestamp will be stored in json format')

src/bin/pg_dump/cdb/cdb_ddboost_util.c

@@ -176,6 +176,7 @@ static int	renameFile(struct ddboost_options *dd_options, ddp_conn_desc_t ddp_co
 static int	setCredential(struct ddboost_options *dd_options);
 static int	copyWithinDDboost(struct ddboost_options *dd_options, ddp_conn_desc_t ddp_conn, int direction);
 static int	get_stream_counts(ddp_conn_desc_t connd, ddp_stream_counts_t * stream_counts);
+static char *getFileWithRegex(struct ddboost_options *dd_options, char *fileWithRegex);
 
 /* Helper routines for handling multiple -T options using a linked list */
 struct schemaTableList *
@@ -1108,23 +1109,121 @@ cleanup:
 int
 readFileFromDDBoost(struct ddboost_options *dd_options)
 {
-	char	   *ddBoostFile = Safe_strdup(dd_options->from_file);
+	char	   *ddboostFile = Safe_strdup(dd_options->from_file);
 	int			err = 0;
 
-	if (!ddBoostFile)
+	if (!ddboostFile)
 	{
 		mpp_err_msg(logError, progname, "Destination file on DDboost not specified\n");
 		return -1;
 	}
+	if (strstr(ddboostFile, "-?[0-9]+")) {
 
-	err = readFromDDFileToOutput(ddBoostFile, dd_options->ddboost_storage_unit);
-	if (ddBoostFile)
-		free(ddBoostFile);
+		char* searchFile = getFileWithRegex(dd_options, ddboostFile);
+		if (searchFile == NULL) {
+			mpp_err_msg(logError, progname, "File pattern %s not found on DDboost\n", ddboostFile);
+			return -1;
+		}
+		ddboostFile = searchFile;
+		mpp_err_msg("DEBUG", progname, "Found file %s\n", ddboostFile);
+	}
+
+	err = readFromDDFileToOutput(ddboostFile, dd_options->ddboost_storage_unit);
+	if (ddboostFile)
+		free(ddboostFile);
 
 	return err;
 
 }
 
+
+char*
+getFileWithRegex(struct ddboost_options *dd_options, char* pathWithRegex)
+{
+	char *lastSlash = NULL;
+	char *ddboostDir = NULL;
+	lastSlash = strrchr(pathWithRegex, '/');
+	ddboostDir = strndup(pathWithRegex, lastSlash - pathWithRegex);
+
+	regex_t regexCompiled;
+	if (regcomp(&regexCompiled, lastSlash + 1, REG_EXTENDED))
+	{
+		printf("Could not compile regular expression.\n");
+		exit(1);
+	}
+
+	int err = 0;
+	if (!ddboostDir)
+	{
+		mpp_err_msg(logError, progname, "Directory on DDboost is not specified\n");
+		err = -1;
+		goto cleanup;
+	}
+
+	ddp_path_t path1 = {0};
+	ddp_dir_desc_t dird = DDP_INVALID_DESCRIPTOR;
+	ddp_dirent_t ret_dirent;
+	path1.su_name = dd_options->ddboost_storage_unit;
+	path1.path_name = ddboostDir;
+
+	err = ddp_open_dir(ddp_conn, &path1, &dird);
+	if (err)
+	{
+		mpp_err_msg(logError, progname, "Opening directory %s on ddboost failed. Err %d\n", ddboostDir, err);
+		err = -1;
+		goto cleanup;
+	}
+	int status = 0;
+	char* filename;
+	while (1)
+	{
+		memset(&ret_dirent, 0, sizeof(ddp_dirent_t));
+		err = ddp_readdir(dird, &ret_dirent);
+		if (err != DD_OK)
+		{
+			if (err == DD_ERR_EMPTY)
+			{
+				err = DD_OK;
+			}
+			else
+			{
+				mpp_err_msg(logError, progname, "Reading directory %s on ddboost failed. Err %d\n", ddboostDir, err);
+			}
+			break;
+		}
+
+		else
+		{
+			status = regexec(&regexCompiled, ret_dirent.d_name, (size_t) 0, NULL, 0);
+			if (status == 0) {
+				filename = malloc(strlen(ddboostDir) + 1 + strlen(ret_dirent.d_name) + 1);
+				strcpy(filename, ddboostDir);
+				strcat(filename, "/");
+				strcat(filename, ret_dirent.d_name);
+				if (dird != DDP_INVALID_DESCRIPTOR)
+				{
+					ddp_close_dir(dird);
+				}
+				if (ddboostDir)
+					free(ddboostDir);
+				return filename;
+			}
+		}
+	}
+
+cleanup:
+    if (dird != DDP_INVALID_DESCRIPTOR)
+    {
+	    ddp_close_dir(dird);
+    }
+    if (ddboostDir)
+        free(ddboostDir);
+	if (err != DD_OK) {
+		exit(1);
+	}
+	return NULL;
+}
+
 int
 deleteDir(struct ddboost_options *dd_options, ddp_conn_desc_t ddp_conn)
 {

src/bin/pg_dump/cdb/cdb_restore_agent.c

@@ -1432,21 +1432,23 @@ static char *formDDBoostFileName(char *pszBackupKey, bool isPostData, char *dd_b
 	/* First form the prefix */
 	char	szFileNamePrefix[1 + MAXPGPATH];
 	int	 instid; /* dispatch node */
-	int	 segid;
+	char	segid[100];
 	int	 len = 0;
 	char	*pszBackupFileName;
 	char 	*dir_name = "db_dumps"; /* default directory */
 
 	instid = g_sourceContentID;
-	segid = g_sourceDBID;
-	if (g_is_old_format)
+	sprintf(segid, "%s", "-?[0-9]+");
+	if (g_is_old_format) {
 		instid = (instid == -1) ? 1 : 0;
+		sprintf(segid, "%d", g_sourceDBID);
+	}
 
 	memset(szFileNamePrefix, 0, (1 + MAXPGPATH));
 	if (dd_boost_dir)
-		snprintf(szFileNamePrefix, 1 + MAXPGPATH, "%s/%sgp_dump_%d_%d_", dd_boost_dir, DUMP_PREFIX, instid, segid);
+		snprintf(szFileNamePrefix, 1 + MAXPGPATH, "%s/%sgp_dump_%d_%s_", dd_boost_dir, DUMP_PREFIX, instid, segid);
 	else
-		snprintf(szFileNamePrefix, 1 + MAXPGPATH, "%s/%sgp_dump_%d_%d_", dir_name, DUMP_PREFIX, instid, segid);
+		snprintf(szFileNamePrefix, 1 + MAXPGPATH, "%s/%sgp_dump_%d_%s_", dir_name, DUMP_PREFIX, instid, segid);
 
 	/* Now add up the length of the pieces */
 	len += strlen(szFileNamePrefix);